blob: 10d0596433f89b849f91c164d7252a29bcac921b [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_PARAVIRT_H
2#define _ASM_X86_PARAVIRT_H
Rusty Russelld3561b72006-12-07 02:14:07 +01003/* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +02005
6#ifdef CONFIG_PARAVIRT
Jeremy Fitzhardinge54321d92009-02-11 10:20:05 -08007#include <asm/pgtable_types.h>
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01008#include <asm/asm.h>
Rusty Russelld3561b72006-12-07 02:14:07 +01009
Jeremy Fitzhardingeac5672f2009-04-14 14:29:44 -070010#include <asm/paravirt_types.h>
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -080011
Rusty Russelld3561b72006-12-07 02:14:07 +010012#ifndef __ASSEMBLY__
Paul Gortmaker187f1882011-11-23 20:12:59 -050013#include <linux/bug.h>
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020014#include <linux/types.h>
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +020015#include <linux/cpumask.h>
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +020016
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020017static inline int paravirt_enabled(void)
18{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070019 return pv_info.paravirt_enabled;
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020020}
Rusty Russelld3561b72006-12-07 02:14:07 +010021
H. Peter Anvinfaca6222008-01-30 13:31:02 +010022static inline void load_sp0(struct tss_struct *tss,
Rusty Russelld3561b72006-12-07 02:14:07 +010023 struct thread_struct *thread)
24{
H. Peter Anvinfaca6222008-01-30 13:31:02 +010025 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
Rusty Russelld3561b72006-12-07 02:14:07 +010026}
27
Rusty Russelld3561b72006-12-07 02:14:07 +010028/* The paravirtualized CPUID instruction. */
29static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
30 unsigned int *ecx, unsigned int *edx)
31{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070032 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
Rusty Russelld3561b72006-12-07 02:14:07 +010033}
34
35/*
36 * These special macros can be used to get or set a debugging register
37 */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020038static inline unsigned long paravirt_get_debugreg(int reg)
39{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070040 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020041}
42#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
43static inline void set_debugreg(unsigned long val, int reg)
44{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070045 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020046}
Rusty Russelld3561b72006-12-07 02:14:07 +010047
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020048static inline void clts(void)
49{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070050 PVOP_VCALL0(pv_cpu_ops.clts);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020051}
Rusty Russelld3561b72006-12-07 02:14:07 +010052
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020053static inline unsigned long read_cr0(void)
54{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070055 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020056}
Rusty Russelld3561b72006-12-07 02:14:07 +010057
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020058static inline void write_cr0(unsigned long x)
59{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070060 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020061}
Rusty Russelld3561b72006-12-07 02:14:07 +010062
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020063static inline unsigned long read_cr2(void)
64{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070065 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020066}
Rusty Russelld3561b72006-12-07 02:14:07 +010067
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020068static inline void write_cr2(unsigned long x)
69{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070070 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020071}
Rusty Russelld3561b72006-12-07 02:14:07 +010072
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020073static inline unsigned long read_cr3(void)
74{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070075 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020076}
77
78static inline void write_cr3(unsigned long x)
79{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070080 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020081}
82
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070083static inline unsigned long __read_cr4(void)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020084{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070085 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020086}
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070087static inline unsigned long __read_cr4_safe(void)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020088{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070089 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020090}
91
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070092static inline void __write_cr4(unsigned long x)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020093{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070094 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020095}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020096
Glauber de Oliveira Costa94ea03c2008-01-30 13:33:19 +010097#ifdef CONFIG_X86_64
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +010098static inline unsigned long read_cr8(void)
99{
100 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
101}
102
103static inline void write_cr8(unsigned long x)
104{
105 PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
106}
Glauber de Oliveira Costa94ea03c2008-01-30 13:33:19 +0100107#endif
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +0100108
David Howellsdf9ee292010-10-07 14:08:55 +0100109static inline void arch_safe_halt(void)
Rusty Russelld3561b72006-12-07 02:14:07 +0100110{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700111 PVOP_VCALL0(pv_irq_ops.safe_halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100112}
113
114static inline void halt(void)
115{
Cliff Wickmanc8217b82010-12-13 10:51:57 -0600116 PVOP_VCALL0(pv_irq_ops.halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100117}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200118
119static inline void wbinvd(void)
120{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700121 PVOP_VCALL0(pv_cpu_ops.wbinvd);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200122}
Rusty Russelld3561b72006-12-07 02:14:07 +0100123
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700124#define get_kernel_rpl() (pv_info.kernel_rpl)
Rusty Russelld3561b72006-12-07 02:14:07 +0100125
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200126static inline u64 paravirt_read_msr(unsigned msr, int *err)
127{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700128 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200129}
Borislav Petkov132ec922009-08-31 09:50:09 +0200130
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200131static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
132{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700133 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200134}
135
Rusty Russell90a0a062007-05-02 19:27:10 +0200136/* These should all do BUG_ON(_err), but our headers are too tangled. */
Joe Perches49cd7402008-03-23 01:03:00 -0700137#define rdmsr(msr, val1, val2) \
138do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200139 int _err; \
140 u64 _l = paravirt_read_msr(msr, &_err); \
141 val1 = (u32)_l; \
142 val2 = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700143} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100144
Joe Perches49cd7402008-03-23 01:03:00 -0700145#define wrmsr(msr, val1, val2) \
146do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200147 paravirt_write_msr(msr, val1, val2); \
Joe Perches49cd7402008-03-23 01:03:00 -0700148} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100149
Joe Perches49cd7402008-03-23 01:03:00 -0700150#define rdmsrl(msr, val) \
151do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200152 int _err; \
153 val = paravirt_read_msr(msr, &_err); \
Joe Perches49cd7402008-03-23 01:03:00 -0700154} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100155
Andy Lutomirski47edb652015-07-23 12:14:40 -0700156static inline void wrmsrl(unsigned msr, u64 val)
157{
158 wrmsr(msr, (u32)val, (u32)(val>>32));
159}
160
Joe Perches49cd7402008-03-23 01:03:00 -0700161#define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b)
Rusty Russelld3561b72006-12-07 02:14:07 +0100162
163/* rdmsr with exception handling */
Joe Perches49cd7402008-03-23 01:03:00 -0700164#define rdmsr_safe(msr, a, b) \
165({ \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200166 int _err; \
167 u64 _l = paravirt_read_msr(msr, &_err); \
168 (*a) = (u32)_l; \
169 (*b) = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700170 _err; \
171})
Rusty Russelld3561b72006-12-07 02:14:07 +0100172
Andi Kleen1de87bd2008-03-22 10:59:28 +0100173static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
174{
175 int err;
176
177 *p = paravirt_read_msr(msr, &err);
178 return err;
179}
Borislav Petkov177fed12009-08-31 09:50:10 +0200180
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700181static inline unsigned long long paravirt_sched_clock(void)
182{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700183 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700184}
Zachary Amsden6cb9a832007-03-05 00:30:35 -0800185
Ingo Molnarc5905af2012-02-24 08:31:31 +0100186struct static_key;
187extern struct static_key paravirt_steal_enabled;
188extern struct static_key paravirt_steal_rq_enabled;
Glauber Costa3c404b52011-07-11 15:28:15 -0400189
190static inline u64 paravirt_steal_clock(int cpu)
191{
192 return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
193}
194
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200195static inline unsigned long long paravirt_read_pmc(int counter)
196{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700197 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200198}
199
Joe Perches49cd7402008-03-23 01:03:00 -0700200#define rdpmc(counter, low, high) \
201do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200202 u64 _l = paravirt_read_pmc(counter); \
203 low = (u32)_l; \
204 high = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700205} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100206
Andi Kleen1ff4d582012-06-05 17:56:50 -0700207#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))
208
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700209static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
210{
211 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
212}
213
214static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
215{
216 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
217}
218
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200219static inline void load_TR_desc(void)
220{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700221 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200222}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100223static inline void load_gdt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200224{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700225 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200226}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100227static inline void load_idt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200228{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700229 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200230}
231static inline void set_ldt(const void *addr, unsigned entries)
232{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700233 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200234}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100235static inline void store_idt(struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200236{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700237 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200238}
239static inline unsigned long paravirt_store_tr(void)
240{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700241 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200242}
243#define store_tr(tr) ((tr) = paravirt_store_tr())
244static inline void load_TLS(struct thread_struct *t, unsigned cpu)
245{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700246 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200247}
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100248
Jeremy Fitzhardinge9f9d4892008-06-25 00:19:32 -0400249#ifdef CONFIG_X86_64
250static inline void load_gs_index(unsigned int gs)
251{
252 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
253}
254#endif
255
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100256static inline void write_ldt_entry(struct desc_struct *dt, int entry,
257 const void *desc)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200258{
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100259 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200260}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100261
262static inline void write_gdt_entry(struct desc_struct *dt, int entry,
263 void *desc, int type)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200264{
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100265 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200266}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100267
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100268static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200269{
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100270 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200271}
272static inline void set_iopl_mask(unsigned mask)
273{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700274 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200275}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200276
Rusty Russelld3561b72006-12-07 02:14:07 +0100277/* The paravirtualized I/O functions */
Joe Perches49cd7402008-03-23 01:03:00 -0700278static inline void slow_down_io(void)
279{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700280 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100281#ifdef REALLY_SLOW_IO
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700282 pv_cpu_ops.io_delay();
283 pv_cpu_ops.io_delay();
284 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100285#endif
286}
287
Zachary Amsdenae5da272007-02-13 13:26:21 +0100288#ifdef CONFIG_SMP
289static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
290 unsigned long start_esp)
291{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700292 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
293 phys_apicid, start_eip, start_esp);
Zachary Amsdenae5da272007-02-13 13:26:21 +0100294}
295#endif
Rusty Russell13623d72006-12-07 02:14:08 +0100296
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200297static inline void paravirt_activate_mm(struct mm_struct *prev,
298 struct mm_struct *next)
299{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700300 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200301}
302
Dave Hansena1ea1c02014-11-18 10:23:49 -0800303static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
304 struct mm_struct *mm)
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200305{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700306 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200307}
308
Dave Hansena1ea1c02014-11-18 10:23:49 -0800309static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200310{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700311 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200312}
313
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200314static inline void __flush_tlb(void)
315{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700316 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200317}
318static inline void __flush_tlb_global(void)
319{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700320 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200321}
322static inline void __flush_tlb_single(unsigned long addr)
323{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700324 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200325}
Rusty Russellda181a82006-12-07 02:14:08 +0100326
Rusty Russell4595f962009-01-10 21:58:09 -0800327static inline void flush_tlb_others(const struct cpumask *cpumask,
328 struct mm_struct *mm,
Alex Shie7b52ff2012-06-28 09:02:17 +0800329 unsigned long start,
330 unsigned long end)
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200331{
Alex Shie7b52ff2012-06-28 09:02:17 +0800332 PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200333}
334
Jeremy Fitzhardingeeba00452008-06-25 00:19:12 -0400335static inline int paravirt_pgd_alloc(struct mm_struct *mm)
336{
337 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
338}
339
340static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
341{
342 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
343}
344
Eduardo Habkostf8639932008-07-30 18:32:27 -0300345static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200346{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700347 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200348}
Eduardo Habkostf8639932008-07-30 18:32:27 -0300349static inline void paravirt_release_pte(unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200350{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700351 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200352}
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100353
Eduardo Habkostf8639932008-07-30 18:32:27 -0300354static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200355{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700356 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200357}
358
Eduardo Habkostf8639932008-07-30 18:32:27 -0300359static inline void paravirt_release_pmd(unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200360{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700361 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200362}
363
Eduardo Habkostf8639932008-07-30 18:32:27 -0300364static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700365{
366 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
367}
Eduardo Habkostf8639932008-07-30 18:32:27 -0300368static inline void paravirt_release_pud(unsigned long pfn)
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700369{
370 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
371}
372
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200373static inline void pte_update(struct mm_struct *mm, unsigned long addr,
374 pte_t *ptep)
375{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700376 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200377}
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800378static inline void pmd_update(struct mm_struct *mm, unsigned long addr,
379 pmd_t *pmdp)
380{
381 PVOP_VCALL3(pv_mmu_ops.pmd_update, mm, addr, pmdp);
382}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200383
384static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
385 pte_t *ptep)
386{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700387 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200388}
389
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800390static inline void pmd_update_defer(struct mm_struct *mm, unsigned long addr,
391 pmd_t *pmdp)
392{
393 PVOP_VCALL3(pv_mmu_ops.pmd_update_defer, mm, addr, pmdp);
394}
395
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100396static inline pte_t __pte(pteval_t val)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200397{
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100398 pteval_t ret;
399
400 if (sizeof(pteval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800401 ret = PVOP_CALLEE2(pteval_t,
402 pv_mmu_ops.make_pte,
403 val, (u64)val >> 32);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100404 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800405 ret = PVOP_CALLEE1(pteval_t,
406 pv_mmu_ops.make_pte,
407 val);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100408
Jeremy Fitzhardingec8e53932008-01-30 13:32:57 +0100409 return (pte_t) { .pte = ret };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200410}
411
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100412static inline pteval_t pte_val(pte_t pte)
413{
414 pteval_t ret;
415
416 if (sizeof(pteval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800417 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
418 pte.pte, (u64)pte.pte >> 32);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100419 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800420 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
421 pte.pte);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100422
423 return ret;
424}
425
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100426static inline pgd_t __pgd(pgdval_t val)
427{
428 pgdval_t ret;
429
430 if (sizeof(pgdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800431 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
432 val, (u64)val >> 32);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100433 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800434 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
435 val);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100436
437 return (pgd_t) { ret };
438}
439
440static inline pgdval_t pgd_val(pgd_t pgd)
441{
442 pgdval_t ret;
443
444 if (sizeof(pgdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800445 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
446 pgd.pgd, (u64)pgd.pgd >> 32);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100447 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800448 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
449 pgd.pgd);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100450
451 return ret;
452}
453
Jeremy Fitzhardinge08b882c2008-06-16 04:30:01 -0700454#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
455static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
456 pte_t *ptep)
457{
458 pteval_t ret;
459
460 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
461 mm, addr, ptep);
462
463 return (pte_t) { .pte = ret };
464}
465
466static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
467 pte_t *ptep, pte_t pte)
468{
469 if (sizeof(pteval_t) > sizeof(long))
470 /* 5 arg words */
471 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
472 else
473 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
474 mm, addr, ptep, pte.pte);
475}
476
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100477static inline void set_pte(pte_t *ptep, pte_t pte)
478{
479 if (sizeof(pteval_t) > sizeof(long))
480 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
481 pte.pte, (u64)pte.pte >> 32);
482 else
483 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
484 pte.pte);
485}
486
487static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
488 pte_t *ptep, pte_t pte)
489{
490 if (sizeof(pteval_t) > sizeof(long))
491 /* 5 arg words */
492 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
493 else
494 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
495}
496
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800497static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
498 pmd_t *pmdp, pmd_t pmd)
499{
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800500 if (sizeof(pmdval_t) > sizeof(long))
501 /* 5 arg words */
502 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd);
503 else
Andrea Arcangelicacf0612011-01-25 15:07:09 -0800504 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp,
505 native_pmd_val(pmd));
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800506}
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800507
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +0100508static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
509{
510 pmdval_t val = native_pmd_val(pmd);
511
512 if (sizeof(pmdval_t) > sizeof(long))
513 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
514 else
515 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
516}
517
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700518#if CONFIG_PGTABLE_LEVELS >= 3
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100519static inline pmd_t __pmd(pmdval_t val)
520{
521 pmdval_t ret;
522
523 if (sizeof(pmdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800524 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
525 val, (u64)val >> 32);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100526 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800527 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
528 val);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100529
530 return (pmd_t) { ret };
531}
532
533static inline pmdval_t pmd_val(pmd_t pmd)
534{
535 pmdval_t ret;
536
537 if (sizeof(pmdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800538 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
539 pmd.pmd, (u64)pmd.pmd >> 32);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100540 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800541 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
542 pmd.pmd);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100543
544 return ret;
545}
546
547static inline void set_pud(pud_t *pudp, pud_t pud)
548{
549 pudval_t val = native_pud_val(pud);
550
551 if (sizeof(pudval_t) > sizeof(long))
552 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
553 val, (u64)val >> 32);
554 else
555 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
556 val);
557}
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700558#if CONFIG_PGTABLE_LEVELS == 4
Eduardo Habkost90422192008-01-30 13:33:20 +0100559static inline pud_t __pud(pudval_t val)
560{
561 pudval_t ret;
562
563 if (sizeof(pudval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800564 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
565 val, (u64)val >> 32);
Eduardo Habkost90422192008-01-30 13:33:20 +0100566 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800567 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
568 val);
Eduardo Habkost90422192008-01-30 13:33:20 +0100569
570 return (pud_t) { ret };
571}
572
573static inline pudval_t pud_val(pud_t pud)
574{
575 pudval_t ret;
576
577 if (sizeof(pudval_t) > sizeof(long))
Jeremy Fitzhardinge4767afb2009-01-29 01:51:34 -0800578 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
579 pud.pud, (u64)pud.pud >> 32);
Eduardo Habkost90422192008-01-30 13:33:20 +0100580 else
Jeremy Fitzhardinge4767afb2009-01-29 01:51:34 -0800581 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
582 pud.pud);
Eduardo Habkost90422192008-01-30 13:33:20 +0100583
584 return ret;
585}
586
587static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
588{
589 pgdval_t val = native_pgd_val(pgd);
590
591 if (sizeof(pgdval_t) > sizeof(long))
592 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
593 val, (u64)val >> 32);
594 else
595 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
596 val);
597}
598
599static inline void pgd_clear(pgd_t *pgdp)
600{
601 set_pgd(pgdp, __pgd(0));
602}
603
604static inline void pud_clear(pud_t *pudp)
605{
606 set_pud(pudp, __pud(0));
607}
608
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700609#endif /* CONFIG_PGTABLE_LEVELS == 4 */
Eduardo Habkost90422192008-01-30 13:33:20 +0100610
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700611#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100612
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100613#ifdef CONFIG_X86_PAE
614/* Special-case pte-setting operations for PAE, which can't update a
615 64-bit pte atomically */
616static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
617{
618 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
619 pte.pte, pte.pte >> 32);
620}
621
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100622static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
623 pte_t *ptep)
624{
625 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
626}
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +0100627
628static inline void pmd_clear(pmd_t *pmdp)
629{
630 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
631}
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100632#else /* !CONFIG_X86_PAE */
633static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
634{
635 set_pte(ptep, pte);
636}
637
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100638static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
639 pte_t *ptep)
640{
641 set_pte_at(mm, addr, ptep, __pte(0));
642}
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +0100643
644static inline void pmd_clear(pmd_t *pmdp)
645{
646 set_pmd(pmdp, __pmd(0));
647}
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100648#endif /* CONFIG_X86_PAE */
649
Jeremy Fitzhardinge7fd7d832009-02-17 23:24:03 -0800650#define __HAVE_ARCH_START_CONTEXT_SWITCH
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800651static inline void arch_start_context_switch(struct task_struct *prev)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200652{
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800653 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200654}
655
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800656static inline void arch_end_context_switch(struct task_struct *next)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200657{
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800658 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200659}
660
Zachary Amsden9226d122007-02-13 13:26:21 +0100661#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200662static inline void arch_enter_lazy_mmu_mode(void)
663{
Jeremy Fitzhardinge8965c1c02007-10-16 11:51:29 -0700664 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200665}
666
667static inline void arch_leave_lazy_mmu_mode(void)
668{
Jeremy Fitzhardinge8965c1c02007-10-16 11:51:29 -0700669 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200670}
671
Boris Ostrovsky511ba862013-03-23 09:36:36 -0400672static inline void arch_flush_lazy_mmu_mode(void)
673{
674 PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush);
675}
Zachary Amsden9226d122007-02-13 13:26:21 +0100676
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700677static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
Masami Hiramatsu3b3809a2009-04-09 10:55:33 -0700678 phys_addr_t phys, pgprot_t flags)
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700679{
680 pv_mmu_ops.set_fixmap(idx, phys, flags);
681}
682
Jeremy Fitzhardingeb4ecc122009-05-13 17:16:55 -0700683#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
Ingo Molnar4bb689e2008-07-09 14:33:33 +0200684
Ingo Molnar62c7a1e2015-05-11 09:47:23 +0200685#ifdef CONFIG_QUEUED_SPINLOCKS
Peter Zijlstra (Intel)f233f7f2015-04-24 14:56:38 -0400686
687static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
688 u32 val)
689{
690 PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val);
691}
692
693static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
694{
695 PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock);
696}
697
698static __always_inline void pv_wait(u8 *ptr, u8 val)
699{
700 PVOP_VCALL2(pv_lock_ops.wait, ptr, val);
701}
702
703static __always_inline void pv_kick(int cpu)
704{
705 PVOP_VCALL1(pv_lock_ops.kick, cpu);
706}
707
Ingo Molnar62c7a1e2015-05-11 09:47:23 +0200708#else /* !CONFIG_QUEUED_SPINLOCKS */
Peter Zijlstra (Intel)f233f7f2015-04-24 14:56:38 -0400709
Jeremy Fitzhardinge545ac132013-08-09 19:51:49 +0530710static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
711 __ticket_t ticket)
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700712{
Jeremy Fitzhardinge354714d2013-08-09 19:51:55 +0530713 PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket);
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700714}
715
Jeremy Fitzhardinge96f853e2013-08-09 19:51:58 +0530716static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
Jeremy Fitzhardinge545ac132013-08-09 19:51:49 +0530717 __ticket_t ticket)
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700718{
Jeremy Fitzhardinge545ac132013-08-09 19:51:49 +0530719 PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700720}
721
Ingo Molnar62c7a1e2015-05-11 09:47:23 +0200722#endif /* CONFIG_QUEUED_SPINLOCKS */
Peter Zijlstra (Intel)f233f7f2015-04-24 14:56:38 -0400723
724#endif /* SMP && PARAVIRT_SPINLOCKS */
Ingo Molnar4bb689e2008-07-09 14:33:33 +0200725
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100726#ifdef CONFIG_X86_32
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800727#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
728#define PV_RESTORE_REGS "popl %edx; popl %ecx;"
729
730/* save and restore all caller-save registers, except return value */
Jeremy Fitzhardingee584f552009-01-30 23:17:23 -0800731#define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
732#define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800733
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100734#define PV_FLAGS_ARG "0"
735#define PV_EXTRA_CLOBBERS
736#define PV_VEXTRA_CLOBBERS
737#else
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800738/* save and restore all caller-save registers, except return value */
739#define PV_SAVE_ALL_CALLER_REGS \
740 "push %rcx;" \
741 "push %rdx;" \
742 "push %rsi;" \
743 "push %rdi;" \
744 "push %r8;" \
745 "push %r9;" \
746 "push %r10;" \
747 "push %r11;"
748#define PV_RESTORE_ALL_CALLER_REGS \
749 "pop %r11;" \
750 "pop %r10;" \
751 "pop %r9;" \
752 "pop %r8;" \
753 "pop %rdi;" \
754 "pop %rsi;" \
755 "pop %rdx;" \
756 "pop %rcx;"
757
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100758/* We save some registers, but all of them, that's too much. We clobber all
759 * caller saved registers but the argument parameter */
760#define PV_SAVE_REGS "pushq %%rdi;"
761#define PV_RESTORE_REGS "popq %%rdi;"
Jeremy Fitzhardingec24481e2008-07-08 15:07:12 -0700762#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
763#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100764#define PV_FLAGS_ARG "D"
765#endif
766
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800767/*
768 * Generate a thunk around a function which saves all caller-save
769 * registers except for the return value. This allows C functions to
770 * be called from assembler code where fewer than normal registers are
771 * available. It may also help code generation around calls from C
772 * code if the common case doesn't use many registers.
773 *
774 * When a callee is wrapped in a thunk, the caller can assume that all
775 * arg regs and all scratch registers are preserved across the
776 * call. The return value in rax/eax will not be saved, even for void
777 * functions.
778 */
779#define PV_CALLEE_SAVE_REGS_THUNK(func) \
780 extern typeof(func) __raw_callee_save_##func; \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800781 \
782 asm(".pushsection .text;" \
Andi Kleena2e7f0e2013-10-22 09:07:56 -0700783 ".globl __raw_callee_save_" #func " ; " \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800784 "__raw_callee_save_" #func ": " \
785 PV_SAVE_ALL_CALLER_REGS \
786 "call " #func ";" \
787 PV_RESTORE_ALL_CALLER_REGS \
788 "ret;" \
789 ".popsection")
790
791/* Get a reference to a callee-save function */
792#define PV_CALLEE_SAVE(func) \
793 ((struct paravirt_callee_save) { __raw_callee_save_##func })
794
795/* Promise that "func" already uses the right calling convention */
796#define __PV_IS_CALLEE_SAVE(func) \
797 ((struct paravirt_callee_save) { func })
798
Steven Rostedtb5908542010-11-10 22:29:49 -0500799static inline notrace unsigned long arch_local_save_flags(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100800{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700801 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100802}
803
Steven Rostedtb5908542010-11-10 22:29:49 -0500804static inline notrace void arch_local_irq_restore(unsigned long f)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100805{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700806 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100807}
808
Steven Rostedtb5908542010-11-10 22:29:49 -0500809static inline notrace void arch_local_irq_disable(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100810{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700811 PVOP_VCALLEE0(pv_irq_ops.irq_disable);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100812}
813
Steven Rostedtb5908542010-11-10 22:29:49 -0500814static inline notrace void arch_local_irq_enable(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100815{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700816 PVOP_VCALLEE0(pv_irq_ops.irq_enable);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100817}
818
Steven Rostedtb5908542010-11-10 22:29:49 -0500819static inline notrace unsigned long arch_local_irq_save(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100820{
821 unsigned long f;
822
David Howellsdf9ee292010-10-07 14:08:55 +0100823 f = arch_local_save_flags();
824 arch_local_irq_disable();
Rusty Russell139ec7c2006-12-07 02:14:08 +0100825 return f;
826}
827
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700828
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200829/* Make sure as little as possible of this mess escapes. */
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200830#undef PARAVIRT_CALL
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200831#undef __PVOP_CALL
832#undef __PVOP_VCALL
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200833#undef PVOP_VCALL0
834#undef PVOP_CALL0
835#undef PVOP_VCALL1
836#undef PVOP_CALL1
837#undef PVOP_VCALL2
838#undef PVOP_CALL2
839#undef PVOP_VCALL3
840#undef PVOP_CALL3
841#undef PVOP_VCALL4
842#undef PVOP_CALL4
Rusty Russell139ec7c2006-12-07 02:14:08 +0100843
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +0200844extern void default_banner(void);
845
Rusty Russelld3561b72006-12-07 02:14:07 +0100846#else /* __ASSEMBLY__ */
847
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100848#define _PVSITE(ptype, clobbers, ops, word, algn) \
Rusty Russell139ec7c2006-12-07 02:14:08 +0100849771:; \
850 ops; \
851772:; \
852 .pushsection .parainstructions,"a"; \
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100853 .align algn; \
854 word 771b; \
Rusty Russell139ec7c2006-12-07 02:14:08 +0100855 .byte ptype; \
856 .byte 772b-771b; \
857 .short clobbers; \
858 .popsection
859
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100860
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800861#define COND_PUSH(set, mask, reg) \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800862 .if ((~(set)) & mask); push %reg; .endif
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800863#define COND_POP(set, mask, reg) \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800864 .if ((~(set)) & mask); pop %reg; .endif
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800865
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100866#ifdef CONFIG_X86_64
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800867
868#define PV_SAVE_REGS(set) \
869 COND_PUSH(set, CLBR_RAX, rax); \
870 COND_PUSH(set, CLBR_RCX, rcx); \
871 COND_PUSH(set, CLBR_RDX, rdx); \
872 COND_PUSH(set, CLBR_RSI, rsi); \
873 COND_PUSH(set, CLBR_RDI, rdi); \
874 COND_PUSH(set, CLBR_R8, r8); \
875 COND_PUSH(set, CLBR_R9, r9); \
876 COND_PUSH(set, CLBR_R10, r10); \
877 COND_PUSH(set, CLBR_R11, r11)
878#define PV_RESTORE_REGS(set) \
879 COND_POP(set, CLBR_R11, r11); \
880 COND_POP(set, CLBR_R10, r10); \
881 COND_POP(set, CLBR_R9, r9); \
882 COND_POP(set, CLBR_R8, r8); \
883 COND_POP(set, CLBR_RDI, rdi); \
884 COND_POP(set, CLBR_RSI, rsi); \
885 COND_POP(set, CLBR_RDX, rdx); \
886 COND_POP(set, CLBR_RCX, rcx); \
887 COND_POP(set, CLBR_RAX, rax)
888
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +0100889#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100890#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400891#define PARA_INDIRECT(addr) *addr(%rip)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100892#else
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800893#define PV_SAVE_REGS(set) \
894 COND_PUSH(set, CLBR_EAX, eax); \
895 COND_PUSH(set, CLBR_EDI, edi); \
896 COND_PUSH(set, CLBR_ECX, ecx); \
897 COND_PUSH(set, CLBR_EDX, edx)
898#define PV_RESTORE_REGS(set) \
899 COND_POP(set, CLBR_EDX, edx); \
900 COND_POP(set, CLBR_ECX, ecx); \
901 COND_POP(set, CLBR_EDI, edi); \
902 COND_POP(set, CLBR_EAX, eax)
903
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +0100904#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100905#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400906#define PARA_INDIRECT(addr) *%cs:addr
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100907#endif
908
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700909#define INTERRUPT_RETURN \
910 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400911 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
Rusty Russell139ec7c2006-12-07 02:14:08 +0100912
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200913#define DISABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700914 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800915 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400916 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800917 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100918
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200919#define ENABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700920 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800921 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400922 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800923 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100924
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400925#define USERGS_SYSRET32 \
926 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +0100927 CLBR_NONE, \
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400928 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100929
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +0100930#ifdef CONFIG_X86_32
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400931#define GET_CR0_INTO_EAX \
932 push %ecx; push %edx; \
933 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +0200934 pop %edx; pop %ecx
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400935
936#define ENABLE_INTERRUPTS_SYSEXIT \
937 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
938 CLBR_NONE, \
939 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
940
941
942#else /* !CONFIG_X86_32 */
Jeremy Fitzhardingea00394f2008-06-25 00:19:30 -0400943
944/*
945 * If swapgs is used while the userspace stack is still current,
946 * there's no way to call a pvop. The PV replacement *must* be
947 * inlined, or the swapgs instruction must be trapped and emulated.
948 */
949#define SWAPGS_UNSAFE_STACK \
950 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
951 swapgs)
952
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800953/*
954 * Note: swapgs is very special, and in practise is either going to be
955 * implemented with a single "swapgs" instruction or something very
956 * special. Either way, we don't need to save any registers for
957 * it.
958 */
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +0100959#define SWAPGS \
960 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800961 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +0100962 )
963
H. Peter Anvinffc4bc92012-04-18 17:16:48 -0700964#define GET_CR2_INTO_RAX \
965 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
Glauber de Oliveira Costa4a8c4c42008-01-30 13:32:07 +0100966
Jeremy Fitzhardingefab58422008-06-25 00:19:31 -0400967#define PARAVIRT_ADJUST_EXCEPTION_FRAME \
968 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
969 CLBR_NONE, \
970 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
971
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400972#define USERGS_SYSRET64 \
973 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \
Jeremy Fitzhardinged75cd222008-06-25 00:19:26 -0400974 CLBR_NONE, \
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400975 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400976#endif /* CONFIG_X86_32 */
Rusty Russell139ec7c2006-12-07 02:14:08 +0100977
Rusty Russelld3561b72006-12-07 02:14:07 +0100978#endif /* __ASSEMBLY__ */
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +0200979#else /* CONFIG_PARAVIRT */
980# define default_banner x86_init_noop
Dave Hansena1ea1c02014-11-18 10:23:49 -0800981#ifndef __ASSEMBLY__
982static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
983 struct mm_struct *mm)
984{
985}
986
987static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
988{
989}
990#endif /* __ASSEMBLY__ */
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +0200991#endif /* !CONFIG_PARAVIRT */
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700992#endif /* _ASM_X86_PARAVIRT_H */