blob: b9d99e0f82c4f072a15f7cb3fb3434d391281daf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1995 Linus Torvalds
3 *
4 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000
Hiroshi Shimamoto66125382008-01-30 13:31:03 +01006 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * X86-64 port
8 * Andi Kleen.
Ashok Raj76e4f662005-06-25 14:55:00 -07009 *
10 * CPU hotplug support - ashok.raj@intel.com
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 */
12
13/*
14 * This file handles the architecture-dependent parts of process handling..
15 */
16
Ashok Raj76e4f662005-06-25 14:55:00 -070017#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/errno.h>
19#include <linux/sched.h>
Hiroshi Shimamoto66125382008-01-30 13:31:03 +010020#include <linux/fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/kernel.h>
22#include <linux/mm.h>
23#include <linux/elfcore.h>
24#include <linux/smp.h>
25#include <linux/slab.h>
26#include <linux/user.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/interrupt.h>
Hiroshi Shimamoto66125382008-01-30 13:31:03 +010028#include <linux/delay.h>
29#include <linux/module.h>
30#include <linux/ptrace.h>
Andi Kleen95833c82006-01-11 22:44:36 +010031#include <linux/notifier.h>
bibo maoc6fd91f2006-03-26 01:38:20 -080032#include <linux/kprobes.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070033#include <linux/kdebug.h>
Erik Bosman529e25f2008-04-14 00:24:18 +020034#include <linux/prctl.h>
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -030035#include <linux/uaccess.h>
36#include <linux/io.h>
Frederic Weisbecker8b96f012008-12-06 03:40:00 +010037#include <linux/ftrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <asm/pgtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <asm/processor.h>
Ingo Molnar78f7f1e2015-04-24 02:54:44 +020041#include <asm/fpu/internal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <asm/mmu_context.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#include <asm/prctl.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <asm/desc.h>
45#include <asm/proto.h>
46#include <asm/ia32.h>
Andi Kleen95833c82006-01-11 22:44:36 +010047#include <asm/idle.h>
Jaswinder Singhbbc1f692008-07-21 21:34:13 +053048#include <asm/syscalls.h>
K.Prasad66cb5912009-06-01 23:44:55 +053049#include <asm/debugreg.h>
David Howellsf05e7982012-03-28 18:11:12 +010050#include <asm/switch_to.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
52asmlinkage extern void ret_from_fork(void);
53
Ingo Molnarc38e5032015-03-17 14:42:59 +010054__visible DEFINE_PER_CPU(unsigned long, rsp_scratch);
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
Hiroshi Shimamoto66125382008-01-30 13:31:03 +010056/* Prints also some state that isn't saved in the pt_regs */
Pekka Enberge2ce07c2008-04-03 16:40:48 +030057void __show_regs(struct pt_regs *regs, int all)
Linus Torvalds1da177e2005-04-16 15:20:36 -070058{
59 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
Alan Sternbb1995d2007-07-21 17:10:42 +020060 unsigned long d0, d1, d2, d3, d6, d7;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +010061 unsigned int fsindex, gsindex;
62 unsigned int ds, cs, es;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Pekka Enbergd015a092009-12-28 10:26:59 +020064 printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
Jiri Slaby5f01c982013-10-25 15:06:58 +020065 printk_address(regs->ip);
Pekka Enbergd015a092009-12-28 10:26:59 +020066 printk(KERN_DEFAULT "RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss,
Gustavo F. Padovan8092c652008-07-29 02:48:52 -030067 regs->sp, regs->flags);
Pekka Enbergd015a092009-12-28 10:26:59 +020068 printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
H. Peter Anvin65ea5b02008-01-30 13:30:56 +010069 regs->ax, regs->bx, regs->cx);
Pekka Enbergd015a092009-12-28 10:26:59 +020070 printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
H. Peter Anvin65ea5b02008-01-30 13:30:56 +010071 regs->dx, regs->si, regs->di);
Pekka Enbergd015a092009-12-28 10:26:59 +020072 printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
H. Peter Anvin65ea5b02008-01-30 13:30:56 +010073 regs->bp, regs->r8, regs->r9);
Pekka Enbergd015a092009-12-28 10:26:59 +020074 printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -030075 regs->r10, regs->r11, regs->r12);
Pekka Enbergd015a092009-12-28 10:26:59 +020076 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -030077 regs->r13, regs->r14, regs->r15);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -030079 asm("movl %%ds,%0" : "=r" (ds));
80 asm("movl %%cs,%0" : "=r" (cs));
81 asm("movl %%es,%0" : "=r" (es));
Linus Torvalds1da177e2005-04-16 15:20:36 -070082 asm("movl %%fs,%0" : "=r" (fsindex));
83 asm("movl %%gs,%0" : "=r" (gsindex));
84
85 rdmsrl(MSR_FS_BASE, fs);
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -030086 rdmsrl(MSR_GS_BASE, gs);
87 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Pekka Enberge2ce07c2008-04-03 16:40:48 +030089 if (!all)
90 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -070091
Glauber de Oliveira Costaf51c9452007-07-22 11:12:29 +020092 cr0 = read_cr0();
93 cr2 = read_cr2();
94 cr3 = read_cr3();
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070095 cr4 = __read_cr4();
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Pekka Enbergd015a092009-12-28 10:26:59 +020097 printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -030098 fs, fsindex, gs, gsindex, shadowgs);
Pekka Enbergd015a092009-12-28 10:26:59 +020099 printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
Gustavo F. Padovan8092c652008-07-29 02:48:52 -0300100 es, cr0);
Pekka Enbergd015a092009-12-28 10:26:59 +0200101 printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
Gustavo F. Padovan8092c652008-07-29 02:48:52 -0300102 cr4);
Alan Sternbb1995d2007-07-21 17:10:42 +0200103
104 get_debugreg(d0, 0);
105 get_debugreg(d1, 1);
106 get_debugreg(d2, 2);
Alan Sternbb1995d2007-07-21 17:10:42 +0200107 get_debugreg(d3, 3);
108 get_debugreg(d6, 6);
109 get_debugreg(d7, 7);
Dave Jones4338774c2013-06-18 12:09:11 -0400110
111 /* Only print out debug registers if they are in their non-default state. */
112 if ((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
113 (d6 == DR6_RESERVED) && (d7 == 0x400))
114 return;
115
116 printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
Pekka Enbergd015a092009-12-28 10:26:59 +0200117 printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
Dave Jones4338774c2013-06-18 12:09:11 -0400118
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119}
120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121void release_thread(struct task_struct *dead_task)
122{
123 if (dead_task->mm) {
Andy Lutomirskia5b9e5a2015-07-30 14:31:34 -0700124#ifdef CONFIG_MODIFY_LDT_SYSCALL
Andy Lutomirski37868fe2015-07-30 14:31:32 -0700125 if (dead_task->mm->context.ldt) {
Chen Gang349eab62012-11-06 14:45:46 +0800126 pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
Joe Perchesc767a542012-05-21 19:50:07 -0700127 dead_task->comm,
Jan Beulich0d430e32015-12-22 08:42:44 -0700128 dead_task->mm->context.ldt->entries,
Andy Lutomirski37868fe2015-07-30 14:31:32 -0700129 dead_task->mm->context.ldt->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 BUG();
131 }
Andy Lutomirskia5b9e5a2015-07-30 14:31:34 -0700132#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 }
134}
135
136static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
137{
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100138 struct user_desc ud = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 .base_addr = addr,
140 .limit = 0xfffff,
141 .seg_32bit = 1,
142 .limit_in_pages = 1,
143 .useable = 1,
144 };
Jan Engelhardtade1af72008-01-30 13:33:23 +0100145 struct desc_struct *desc = t->thread.tls_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 desc += tls;
Glauber de Oliveira Costa80fbb692008-01-30 13:31:13 +0100147 fill_ldt(desc, &ud);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148}
149
150static inline u32 read_32bit_tls(struct task_struct *t, int tls)
151{
Roland McGrath91394eb2008-01-30 13:30:45 +0100152 return get_desc_base(&t->thread.tls_array[tls]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153}
154
Josh Triplettc1bd55f2015-06-30 15:00:00 -0700155int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
156 unsigned long arg, struct task_struct *p, unsigned long tls)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157{
158 int err;
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300159 struct pt_regs *childregs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 struct task_struct *me = current;
161
Al Viro7076aad2012-09-10 16:44:54 -0400162 p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
163 childregs = task_pt_regs(p);
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100164 p->thread.sp = (unsigned long) childregs;
Al Viroe4f17c42006-01-12 01:05:38 -0800165 set_tsk_thread_flag(p, TIF_FORK);
K.Prasad66cb5912009-06-01 23:44:55 +0530166 p->thread.io_bitmap_ptr = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400168 savesegment(gs, p->thread.gsindex);
H. Peter Anvin7ce5a2b2010-04-23 16:17:40 -0700169 p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400170 savesegment(fs, p->thread.fsindex);
H. Peter Anvin7ce5a2b2010-04-23 16:17:40 -0700171 p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400172 savesegment(es, p->thread.es);
173 savesegment(ds, p->thread.ds);
Al Viro7076aad2012-09-10 16:44:54 -0400174 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
175
Al Viro1d4b4b22012-10-22 22:34:11 -0400176 if (unlikely(p->flags & PF_KTHREAD)) {
Al Viro7076aad2012-09-10 16:44:54 -0400177 /* kernel thread */
178 memset(childregs, 0, sizeof(struct pt_regs));
179 childregs->sp = (unsigned long)childregs;
180 childregs->ss = __KERNEL_DS;
181 childregs->bx = sp; /* function */
182 childregs->bp = arg;
183 childregs->orig_ax = -1;
184 childregs->cs = __KERNEL_CS | get_kernel_rpl();
H. Peter Anvin1adfa762013-04-27 16:10:11 -0700185 childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_FIXED;
Al Viro7076aad2012-09-10 16:44:54 -0400186 return 0;
187 }
Al Viro1d4b4b22012-10-22 22:34:11 -0400188 *childregs = *current_pt_regs();
Al Viro7076aad2012-09-10 16:44:54 -0400189
190 childregs->ax = 0;
Al Viro1d4b4b22012-10-22 22:34:11 -0400191 if (sp)
192 childregs->sp = sp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
K.Prasad66cb5912009-06-01 23:44:55 +0530194 err = -ENOMEM;
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200195 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
Thomas Meyercced4022011-11-17 23:43:40 +0100196 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
197 IO_BITMAP_BYTES, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 if (!p->thread.io_bitmap_ptr) {
199 p->thread.io_bitmap_max = 0;
200 return -ENOMEM;
201 }
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200202 set_tsk_thread_flag(p, TIF_IO_BITMAP);
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100203 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204
205 /*
206 * Set a new TLS for the child thread?
207 */
208 if (clone_flags & CLONE_SETTLS) {
209#ifdef CONFIG_IA32_EMULATION
Andy Lutomirski72c6fb42015-02-24 16:01:39 -0800210 if (is_ia32_task())
Roland McGrathefd1ca52008-01-30 13:30:46 +0100211 err = do_set_thread_area(p, -1,
Josh Triplettc1bd55f2015-06-30 15:00:00 -0700212 (struct user_desc __user *)tls, 0);
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300213 else
214#endif
Josh Triplettc1bd55f2015-06-30 15:00:00 -0700215 err = do_arch_prctl(p, ARCH_SET_FS, tls);
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300216 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 goto out;
218 }
219 err = 0;
220out:
221 if (err && p->thread.io_bitmap_ptr) {
222 kfree(p->thread.io_bitmap_ptr);
223 p->thread.io_bitmap_max = 0;
224 }
K.Prasad66cb5912009-06-01 23:44:55 +0530225
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 return err;
227}
228
H. Peter Anvine634d8f2009-10-09 15:56:53 -0700229static void
230start_thread_common(struct pt_regs *regs, unsigned long new_ip,
231 unsigned long new_sp,
232 unsigned int _cs, unsigned int _ss, unsigned int _ds)
Ingo Molnar513ad842008-02-21 05:18:40 +0100233{
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400234 loadsegment(fs, 0);
H. Peter Anvine634d8f2009-10-09 15:56:53 -0700235 loadsegment(es, _ds);
236 loadsegment(ds, _ds);
Ingo Molnar513ad842008-02-21 05:18:40 +0100237 load_gs_index(0);
238 regs->ip = new_ip;
239 regs->sp = new_sp;
H. Peter Anvine634d8f2009-10-09 15:56:53 -0700240 regs->cs = _cs;
241 regs->ss = _ss;
H. Peter Anvina6f05a62009-10-08 18:02:54 -0700242 regs->flags = X86_EFLAGS_IF;
Brian Gerst1daeaa32015-03-21 18:54:21 -0400243 force_iret();
Ingo Molnar513ad842008-02-21 05:18:40 +0100244}
H. Peter Anvine634d8f2009-10-09 15:56:53 -0700245
246void
247start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
248{
249 start_thread_common(regs, new_ip, new_sp,
250 __USER_CS, __USER_DS, 0);
251}
Ingo Molnar513ad842008-02-21 05:18:40 +0100252
Brian Gerst7da77072015-06-22 07:55:13 -0400253#ifdef CONFIG_COMPAT
254void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp)
H. Peter Anvina6f05a62009-10-08 18:02:54 -0700255{
H. Peter Anvine634d8f2009-10-09 15:56:53 -0700256 start_thread_common(regs, new_ip, new_sp,
H. Peter Anvind1a797f2012-02-19 10:06:34 -0800257 test_thread_flag(TIF_X32)
258 ? __USER_CS : __USER32_CS,
259 __USER_DS, __USER_DS);
H. Peter Anvina6f05a62009-10-08 18:02:54 -0700260}
261#endif
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263/*
264 * switch_to(x,y) should switch tasks from x to y.
265 *
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100266 * This could still be optimized:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 * - fold all the options into a flag word and test it with a single test.
268 * - could test fs/gs bitsliced
Andi Kleen099f3182006-02-03 21:51:38 +0100269 *
270 * Kprobes not supported here. Set the probe on schedule instead.
Frederic Weisbecker8b96f012008-12-06 03:40:00 +0100271 * Function graph tracer not supported too.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 */
Andi Kleen35ea79032013-08-05 15:02:39 -0700273__visible __notrace_funcgraph struct task_struct *
Andi Kleena88cde12005-11-05 17:25:54 +0100274__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700276 struct thread_struct *prev = &prev_p->thread;
277 struct thread_struct *next = &next_p->thread;
Ingo Molnar384a23f2015-04-23 17:43:27 +0200278 struct fpu *prev_fpu = &prev->fpu;
279 struct fpu *next_fpu = &next->fpu;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100280 int cpu = smp_processor_id();
Andy Lutomirski24933b82015-03-05 19:19:05 -0800281 struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
Jeremy Fitzhardinge478de5a2008-06-25 00:19:24 -0400282 unsigned fsindex, gsindex;
Ingo Molnar384a23f2015-04-23 17:43:27 +0200283 fpu_switch_t fpu_switch;
Arjan van de Vene07e23e2006-09-26 10:52:36 +0200284
Ingo Molnar384a23f2015-04-23 17:43:27 +0200285 fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
Linus Torvalds49030622012-02-16 19:11:15 -0800286
Jeremy Fitzhardinge478de5a2008-06-25 00:19:24 -0400287 /* We must save %fs and %gs before load_TLS() because
288 * %fs and %gs may be cleared by load_TLS().
289 *
290 * (e.g. xen_load_tls())
291 */
292 savesegment(fs, fsindex);
293 savesegment(gs, gsindex);
294
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800295 /*
296 * Load TLS before restoring any segments so that segment loads
297 * reference the correct GDT entries.
298 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 load_TLS(next, cpu);
300
Jeremy Fitzhardinge3fe0a632008-06-25 00:19:23 -0400301 /*
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800302 * Leave lazy mode, flushing any hypercalls made here. This
303 * must be done after loading TLS entries in the GDT but before
304 * loading segments that might reference them, and and it must
Ingo Molnar3a0aee42015-04-22 13:16:47 +0200305 * be done before fpu__restore(), so the TS bit is up to
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800306 * date.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 */
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800308 arch_end_context_switch(next_p);
Jeremy Fitzhardinge3fe0a632008-06-25 00:19:23 -0400309
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800310 /* Switch DS and ES.
311 *
312 * Reading them only returns the selectors, but writing them (if
313 * nonzero) loads the full descriptor from the GDT or LDT. The
314 * LDT for next is loaded in switch_mm, and the GDT is loaded
315 * above.
316 *
317 * We therefore need to write new values to the segment
318 * registers on every context switch unless both the new and old
319 * values are zero.
320 *
321 * Note that we don't need to do anything for CS and SS, as
322 * those are saved and restored as part of pt_regs.
323 */
324 savesegment(es, prev->es);
325 if (unlikely(next->es | prev->es))
326 loadsegment(es, next->es);
327
328 savesegment(ds, prev->ds);
329 if (unlikely(next->ds | prev->ds))
330 loadsegment(ds, next->ds);
331
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300332 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 * Switch FS and GS.
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700334 *
Chuck Ebbert558a65b2015-10-14 14:31:19 -0400335 * These are even more complicated than DS and ES: they have
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800336 * 64-bit bases are that controlled by arch_prctl. Those bases
337 * only differ from the values in the GDT or LDT if the selector
338 * is 0.
339 *
340 * Loading the segment register resets the hidden base part of
341 * the register to 0 or the value from the GDT / LDT. If the
342 * next base address zero, writing 0 to the segment register is
343 * much faster than using wrmsr to explicitly zero the base.
344 *
345 * The thread_struct.fs and thread_struct.gs values are 0
346 * if the fs and gs bases respectively are not overridden
347 * from the values implied by fsindex and gsindex. They
348 * are nonzero, and store the nonzero base addresses, if
349 * the bases are overridden.
350 *
351 * (fs != 0 && fsindex != 0) || (gs != 0 && gsindex != 0) should
352 * be impossible.
353 *
354 * Therefore we need to reload the segment registers if either
355 * the old or new selector is nonzero, and we need to override
356 * the base address if next thread expects it to be overridden.
357 *
358 * This code is unnecessarily slow in the case where the old and
359 * new indexes are zero and the new base is nonzero -- it will
360 * unnecessarily write 0 to the selector before writing the new
361 * base address.
362 *
363 * Note: This all depends on arch_prctl being the only way that
364 * user code can override the segment base. Once wrfsbase and
365 * wrgsbase are enabled, most of this code will need to change.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 */
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700367 if (unlikely(fsindex | next->fsindex | prev->fs)) {
368 loadsegment(fs, next->fsindex);
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800369
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300370 /*
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800371 * If user code wrote a nonzero value to FS, then it also
372 * cleared the overridden base address.
373 *
374 * XXX: if user code wrote 0 to FS and cleared the base
375 * address itself, we won't notice and we'll incorrectly
376 * restore the prior base address next time we reschdule
377 * the process.
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700378 */
379 if (fsindex)
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300380 prev->fs = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 }
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700382 if (next->fs)
383 wrmsrl(MSR_FS_BASE, next->fs);
384 prev->fsindex = fsindex;
385
386 if (unlikely(gsindex | next->gsindex | prev->gs)) {
387 load_gs_index(next->gsindex);
Andy Lutomirskif647d7c2014-12-08 13:55:20 -0800388
389 /* This works (and fails) the same way as fsindex above. */
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700390 if (gsindex)
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300391 prev->gs = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 }
Jeremy Fitzhardinge87b935a2008-07-08 15:06:26 -0700393 if (next->gs)
394 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
395 prev->gsindex = gsindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Ingo Molnar384a23f2015-04-23 17:43:27 +0200397 switch_fpu_finish(next_fpu, fpu_switch);
Linus Torvalds34ddc812012-02-18 12:56:35 -0800398
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300399 /*
Jan Beulich45948d72006-03-25 16:29:25 +0100400 * Switch the PDA and FPU contexts.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 */
Alex Shic6ae41e2012-05-11 15:35:27 +0800402 this_cpu_write(current_task, next_p);
Andi Kleen18bd0572006-04-20 02:36:45 +0200403
Andy Lutomirskib27559a2015-03-06 17:50:18 -0800404 /* Reload esp0 and ss1. This changes current_thread_info(). */
405 load_sp0(tss, next);
406
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 /*
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200408 * Now maybe reload the debug registers and handle I/O bitmaps
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 */
Markus Metzgereee3af42008-01-30 13:31:09 +0100410 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
411 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
Stephane Eraniand3a4f482006-09-26 10:52:28 +0200412 __switch_to_xtra(prev_p, next_p, tss);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Andy Lutomirski61f01dd2015-04-26 16:47:59 -0700414 if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) {
415 /*
416 * AMD CPUs have a misfeature: SYSRET sets the SS selector but
417 * does not update the cached descriptor. As a result, if we
418 * do SYSRET while SS is NULL, we'll end up in user mode with
419 * SS apparently equal to __USER_DS but actually unusable.
420 *
421 * The straightforward workaround would be to fix it up just
422 * before SYSRET, but that would slow down the system call
423 * fast paths. Instead, we ensure that SS is never NULL in
424 * system call context. We do this by replacing NULL SS
425 * selectors at every context switch. SYSCALL sets up a valid
426 * SS, so the only way to get NULL is to re-enter the kernel
427 * from CPL 3 through an interrupt. Since that can't happen
428 * in the same task as a running syscall, we are guaranteed to
429 * context switch between every interrupt vector entry and a
430 * subsequent SYSRET.
431 *
432 * We read SS first because SS reads are much faster than
433 * writes. Out of caution, we force SS to __KERNEL_DS even if
434 * it previously had a different non-NULL value.
435 */
436 unsigned short ss_sel;
437 savesegment(ss, ss_sel);
438 if (ss_sel != __KERNEL_DS)
439 loadsegment(ss, __KERNEL_DS);
440 }
441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 return prev_p;
443}
444
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445void set_personality_64bit(void)
446{
447 /* inherit personality from parent */
448
449 /* Make sure to be in 64bit mode */
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100450 clear_thread_flag(TIF_IA32);
H. Peter Anvin6bd33002012-02-06 13:03:09 -0800451 clear_thread_flag(TIF_ADDR32);
H. Peter Anvinbb212722012-02-14 13:56:49 -0800452 clear_thread_flag(TIF_X32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
Stephen Wilson375906f2011-03-13 15:49:14 -0400454 /* Ensure the corresponding mm is not marked. */
455 if (current->mm)
456 current->mm->context.ia32_compat = 0;
457
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 /* TBD: overwrites user setup. Should have two bits.
459 But 64bit processes have always behaved this way,
460 so it's not too bad. The main problem is just that
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100461 32bit childs are affected again. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 current->personality &= ~READ_IMPLIES_EXEC;
463}
464
H. Peter Anvind1a797f2012-02-19 10:06:34 -0800465void set_personality_ia32(bool x32)
H. Peter Anvin05d43ed2010-01-28 22:14:43 -0800466{
467 /* inherit personality from parent */
468
469 /* Make sure to be in 32bit mode */
H. Peter Anvin6bd33002012-02-06 13:03:09 -0800470 set_thread_flag(TIF_ADDR32);
H. Peter Anvin05d43ed2010-01-28 22:14:43 -0800471
Stephen Wilson375906f2011-03-13 15:49:14 -0400472 /* Mark the associated mm as containing 32-bit tasks. */
H. Peter Anvind1a797f2012-02-19 10:06:34 -0800473 if (x32) {
474 clear_thread_flag(TIF_IA32);
475 set_thread_flag(TIF_X32);
Oleg Nesterovb24dc8d2014-04-19 18:10:09 +0200476 if (current->mm)
477 current->mm->context.ia32_compat = TIF_X32;
H. Peter Anvind1a797f2012-02-19 10:06:34 -0800478 current->personality &= ~READ_IMPLIES_EXEC;
Bobby Powersce5f7a92012-02-25 23:25:38 -0500479 /* is_compat_task() uses the presence of the x32
480 syscall bit flag to determine compat status */
481 current_thread_info()->status &= ~TS_COMPAT;
H. Peter Anvind1a797f2012-02-19 10:06:34 -0800482 } else {
483 set_thread_flag(TIF_IA32);
484 clear_thread_flag(TIF_X32);
Oleg Nesterovb24dc8d2014-04-19 18:10:09 +0200485 if (current->mm)
486 current->mm->context.ia32_compat = TIF_IA32;
H. Peter Anvind1a797f2012-02-19 10:06:34 -0800487 current->personality |= force_personality32;
488 /* Prepare the first "return" to user space */
489 current_thread_info()->status |= TS_COMPAT;
490 }
H. Peter Anvin05d43ed2010-01-28 22:14:43 -0800491}
Larry Fingerfebb72a2012-05-06 19:40:03 -0500492EXPORT_SYMBOL_GPL(set_personality_ia32);
H. Peter Anvin05d43ed2010-01-28 22:14:43 -0800493
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300495{
496 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 int doit = task == current;
498 int cpu;
499
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300500 switch (code) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 case ARCH_SET_GS:
Suresh Siddha84929802005-06-21 17:14:32 -0700502 if (addr >= TASK_SIZE_OF(task))
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300503 return -EPERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 cpu = get_cpu();
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300505 /* handle small bases via the GDT because that's faster to
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 switch. */
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300507 if (addr <= 0xffffffff) {
508 set_32bit_tls(task, GS_TLS, addr);
509 if (doit) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 load_TLS(&task->thread, cpu);
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300511 load_gs_index(GS_TLS_SEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 }
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300513 task->thread.gsindex = GS_TLS_SEL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 task->thread.gs = 0;
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300515 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 task->thread.gsindex = 0;
517 task->thread.gs = addr;
518 if (doit) {
Andi Kleena88cde12005-11-05 17:25:54 +0100519 load_gs_index(0);
H. Peter Anvin715c85b2012-06-07 13:32:04 -0700520 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300521 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 }
523 put_cpu();
524 break;
525 case ARCH_SET_FS:
526 /* Not strictly needed for fs, but do it for symmetry
527 with gs */
Suresh Siddha84929802005-06-21 17:14:32 -0700528 if (addr >= TASK_SIZE_OF(task))
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100529 return -EPERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 cpu = get_cpu();
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100531 /* handle small bases via the GDT because that's faster to
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 switch. */
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100533 if (addr <= 0xffffffff) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 set_32bit_tls(task, FS_TLS, addr);
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100535 if (doit) {
536 load_TLS(&task->thread, cpu);
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400537 loadsegment(fs, FS_TLS_SEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538 }
539 task->thread.fsindex = FS_TLS_SEL;
540 task->thread.fs = 0;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100541 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 task->thread.fsindex = 0;
543 task->thread.fs = addr;
544 if (doit) {
545 /* set the selector to 0 to not confuse
546 __switch_to */
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400547 loadsegment(fs, 0);
H. Peter Anvin715c85b2012-06-07 13:32:04 -0700548 ret = wrmsrl_safe(MSR_FS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 }
550 }
551 put_cpu();
552 break;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100553 case ARCH_GET_FS: {
554 unsigned long base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 if (task->thread.fsindex == FS_TLS_SEL)
556 base = read_32bit_tls(task, FS_TLS);
Andi Kleena88cde12005-11-05 17:25:54 +0100557 else if (doit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 rdmsrl(MSR_FS_BASE, base);
Andi Kleena88cde12005-11-05 17:25:54 +0100559 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 base = task->thread.fs;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100561 ret = put_user(base, (unsigned long __user *)addr);
562 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 }
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100564 case ARCH_GET_GS: {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 unsigned long base;
John Blackwood97c28032006-04-07 19:50:25 +0200566 unsigned gsindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 if (task->thread.gsindex == GS_TLS_SEL)
568 base = read_32bit_tls(task, GS_TLS);
John Blackwood97c28032006-04-07 19:50:25 +0200569 else if (doit) {
Jeremy Fitzhardingeada85702008-06-25 00:19:00 -0400570 savesegment(gs, gsindex);
John Blackwood97c28032006-04-07 19:50:25 +0200571 if (gsindex)
572 rdmsrl(MSR_KERNEL_GS_BASE, base);
573 else
574 base = task->thread.gs;
Gustavo F. Padovan7de08b42008-07-29 02:48:51 -0300575 } else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 base = task->thread.gs;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100577 ret = put_user(base, (unsigned long __user *)addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 break;
579 }
580
581 default:
582 ret = -EINVAL;
583 break;
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100584 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
Hiroshi Shimamoto66125382008-01-30 13:31:03 +0100586 return ret;
587}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588
589long sys_arch_prctl(int code, unsigned long addr)
590{
591 return do_arch_prctl(current, code, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592}
593
Stefani Seibold89240ba2009-11-03 10:22:40 +0100594unsigned long KSTK_ESP(struct task_struct *task)
595{
Denys Vlasenko263042e2015-03-09 19:39:23 +0100596 return task_pt_regs(task)->sp;
Stefani Seibold89240ba2009-11-03 10:22:40 +0100597}