H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 1 | #ifndef _ASM_X86_PARAVIRT_H |
| 2 | #define _ASM_X86_PARAVIRT_H |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 3 | /* Various instructions on x86 need to be replaced for |
| 4 | * para-virtualization: those hooks are defined here. */ |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 5 | |
| 6 | #ifdef CONFIG_PARAVIRT |
Jeremy Fitzhardinge | 54321d9 | 2009-02-11 10:20:05 -0800 | [diff] [blame] | 7 | #include <asm/pgtable_types.h> |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 8 | #include <asm/asm.h> |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 9 | |
Jeremy Fitzhardinge | ac5672f | 2009-04-14 14:29:44 -0700 | [diff] [blame] | 10 | #include <asm/paravirt_types.h> |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 11 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 12 | #ifndef __ASSEMBLY__ |
Paul Gortmaker | 187f188 | 2011-11-23 20:12:59 -0500 | [diff] [blame] | 13 | #include <linux/bug.h> |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 14 | #include <linux/types.h> |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 15 | #include <linux/cpumask.h> |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 16 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 17 | static inline int paravirt_enabled(void) |
| 18 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 19 | return pv_info.paravirt_enabled; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 20 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 21 | |
H. Peter Anvin | faca622 | 2008-01-30 13:31:02 +0100 | [diff] [blame] | 22 | static inline void load_sp0(struct tss_struct *tss, |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 23 | struct thread_struct *thread) |
| 24 | { |
H. Peter Anvin | faca622 | 2008-01-30 13:31:02 +0100 | [diff] [blame] | 25 | PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 26 | } |
| 27 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 28 | /* The paravirtualized CPUID instruction. */ |
| 29 | static inline void __cpuid(unsigned int *eax, unsigned int *ebx, |
| 30 | unsigned int *ecx, unsigned int *edx) |
| 31 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 32 | PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 33 | } |
| 34 | |
| 35 | /* |
| 36 | * These special macros can be used to get or set a debugging register |
| 37 | */ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 38 | static inline unsigned long paravirt_get_debugreg(int reg) |
| 39 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 40 | return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 41 | } |
| 42 | #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) |
| 43 | static inline void set_debugreg(unsigned long val, int reg) |
| 44 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 45 | PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 46 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 47 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 48 | static inline void clts(void) |
| 49 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 50 | PVOP_VCALL0(pv_cpu_ops.clts); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 51 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 52 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 53 | static inline unsigned long read_cr0(void) |
| 54 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 55 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 56 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 57 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 58 | static inline void write_cr0(unsigned long x) |
| 59 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 60 | PVOP_VCALL1(pv_cpu_ops.write_cr0, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 61 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 62 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 63 | static inline unsigned long read_cr2(void) |
| 64 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 65 | return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 66 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 67 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 68 | static inline void write_cr2(unsigned long x) |
| 69 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 70 | PVOP_VCALL1(pv_mmu_ops.write_cr2, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 71 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 72 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 73 | static inline unsigned long read_cr3(void) |
| 74 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 75 | return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 76 | } |
| 77 | |
| 78 | static inline void write_cr3(unsigned long x) |
| 79 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 80 | PVOP_VCALL1(pv_mmu_ops.write_cr3, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 81 | } |
| 82 | |
Andy Lutomirski | 1e02ce4 | 2014-10-24 15:58:08 -0700 | [diff] [blame] | 83 | static inline unsigned long __read_cr4(void) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 84 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 85 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 86 | } |
Andy Lutomirski | 1e02ce4 | 2014-10-24 15:58:08 -0700 | [diff] [blame] | 87 | static inline unsigned long __read_cr4_safe(void) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 88 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 89 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 90 | } |
| 91 | |
Andy Lutomirski | 1e02ce4 | 2014-10-24 15:58:08 -0700 | [diff] [blame] | 92 | static inline void __write_cr4(unsigned long x) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 93 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 94 | PVOP_VCALL1(pv_cpu_ops.write_cr4, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 95 | } |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 96 | |
Glauber de Oliveira Costa | 94ea03c | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 97 | #ifdef CONFIG_X86_64 |
Glauber de Oliveira Costa | 4c9890c | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 98 | static inline unsigned long read_cr8(void) |
| 99 | { |
| 100 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8); |
| 101 | } |
| 102 | |
| 103 | static inline void write_cr8(unsigned long x) |
| 104 | { |
| 105 | PVOP_VCALL1(pv_cpu_ops.write_cr8, x); |
| 106 | } |
Glauber de Oliveira Costa | 94ea03c | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 107 | #endif |
Glauber de Oliveira Costa | 4c9890c | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 108 | |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 109 | static inline void arch_safe_halt(void) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 110 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 111 | PVOP_VCALL0(pv_irq_ops.safe_halt); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 112 | } |
| 113 | |
| 114 | static inline void halt(void) |
| 115 | { |
Cliff Wickman | c8217b8 | 2010-12-13 10:51:57 -0600 | [diff] [blame] | 116 | PVOP_VCALL0(pv_irq_ops.halt); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 117 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 118 | |
| 119 | static inline void wbinvd(void) |
| 120 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 121 | PVOP_VCALL0(pv_cpu_ops.wbinvd); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 122 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 123 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 124 | #define get_kernel_rpl() (pv_info.kernel_rpl) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 125 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 126 | static inline u64 paravirt_read_msr(unsigned msr, int *err) |
| 127 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 128 | return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 129 | } |
Borislav Petkov | 132ec92 | 2009-08-31 09:50:09 +0200 | [diff] [blame] | 130 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 131 | static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) |
| 132 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 133 | return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 134 | } |
| 135 | |
Rusty Russell | 90a0a06 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 136 | /* These should all do BUG_ON(_err), but our headers are too tangled. */ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 137 | #define rdmsr(msr, val1, val2) \ |
| 138 | do { \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 139 | int _err; \ |
| 140 | u64 _l = paravirt_read_msr(msr, &_err); \ |
| 141 | val1 = (u32)_l; \ |
| 142 | val2 = _l >> 32; \ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 143 | } while (0) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 144 | |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 145 | #define wrmsr(msr, val1, val2) \ |
| 146 | do { \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 147 | paravirt_write_msr(msr, val1, val2); \ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 148 | } while (0) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 149 | |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 150 | #define rdmsrl(msr, val) \ |
| 151 | do { \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 152 | int _err; \ |
| 153 | val = paravirt_read_msr(msr, &_err); \ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 154 | } while (0) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 155 | |
Andy Lutomirski | 47edb65 | 2015-07-23 12:14:40 -0700 | [diff] [blame] | 156 | static inline void wrmsrl(unsigned msr, u64 val) |
| 157 | { |
| 158 | wrmsr(msr, (u32)val, (u32)(val>>32)); |
| 159 | } |
| 160 | |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 161 | #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 162 | |
| 163 | /* rdmsr with exception handling */ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 164 | #define rdmsr_safe(msr, a, b) \ |
| 165 | ({ \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 166 | int _err; \ |
| 167 | u64 _l = paravirt_read_msr(msr, &_err); \ |
| 168 | (*a) = (u32)_l; \ |
| 169 | (*b) = _l >> 32; \ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 170 | _err; \ |
| 171 | }) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 172 | |
Andi Kleen | 1de87bd | 2008-03-22 10:59:28 +0100 | [diff] [blame] | 173 | static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) |
| 174 | { |
| 175 | int err; |
| 176 | |
| 177 | *p = paravirt_read_msr(msr, &err); |
| 178 | return err; |
| 179 | } |
Borislav Petkov | 177fed1 | 2009-08-31 09:50:10 +0200 | [diff] [blame] | 180 | |
Jeremy Fitzhardinge | 688340e | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 181 | static inline unsigned long long paravirt_sched_clock(void) |
| 182 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 183 | return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); |
Jeremy Fitzhardinge | 688340e | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 184 | } |
Zachary Amsden | 6cb9a83 | 2007-03-05 00:30:35 -0800 | [diff] [blame] | 185 | |
Ingo Molnar | c5905af | 2012-02-24 08:31:31 +0100 | [diff] [blame] | 186 | struct static_key; |
| 187 | extern struct static_key paravirt_steal_enabled; |
| 188 | extern struct static_key paravirt_steal_rq_enabled; |
Glauber Costa | 3c404b5 | 2011-07-11 15:28:15 -0400 | [diff] [blame] | 189 | |
| 190 | static inline u64 paravirt_steal_clock(int cpu) |
| 191 | { |
| 192 | return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu); |
| 193 | } |
| 194 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 195 | static inline unsigned long long paravirt_read_pmc(int counter) |
| 196 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 197 | return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 198 | } |
| 199 | |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 200 | #define rdpmc(counter, low, high) \ |
| 201 | do { \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 202 | u64 _l = paravirt_read_pmc(counter); \ |
| 203 | low = (u32)_l; \ |
| 204 | high = _l >> 32; \ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 205 | } while (0) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 206 | |
Andi Kleen | 1ff4d58 | 2012-06-05 17:56:50 -0700 | [diff] [blame] | 207 | #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) |
| 208 | |
Jeremy Fitzhardinge | 38ffbe6 | 2008-07-23 14:21:18 -0700 | [diff] [blame] | 209 | static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) |
| 210 | { |
| 211 | PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries); |
| 212 | } |
| 213 | |
| 214 | static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) |
| 215 | { |
| 216 | PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries); |
| 217 | } |
| 218 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 219 | static inline void load_TR_desc(void) |
| 220 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 221 | PVOP_VCALL0(pv_cpu_ops.load_tr_desc); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 222 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 223 | static inline void load_gdt(const struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 224 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 225 | PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 226 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 227 | static inline void load_idt(const struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 228 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 229 | PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 230 | } |
| 231 | static inline void set_ldt(const void *addr, unsigned entries) |
| 232 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 233 | PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 234 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 235 | static inline void store_idt(struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 236 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 237 | PVOP_VCALL1(pv_cpu_ops.store_idt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 238 | } |
| 239 | static inline unsigned long paravirt_store_tr(void) |
| 240 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 241 | return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 242 | } |
| 243 | #define store_tr(tr) ((tr) = paravirt_store_tr()) |
| 244 | static inline void load_TLS(struct thread_struct *t, unsigned cpu) |
| 245 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 246 | PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 247 | } |
Glauber de Oliveira Costa | 75b8bb3 | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 248 | |
Jeremy Fitzhardinge | 9f9d489 | 2008-06-25 00:19:32 -0400 | [diff] [blame] | 249 | #ifdef CONFIG_X86_64 |
| 250 | static inline void load_gs_index(unsigned int gs) |
| 251 | { |
| 252 | PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs); |
| 253 | } |
| 254 | #endif |
| 255 | |
Glauber de Oliveira Costa | 75b8bb3 | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 256 | static inline void write_ldt_entry(struct desc_struct *dt, int entry, |
| 257 | const void *desc) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 258 | { |
Glauber de Oliveira Costa | 75b8bb3 | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 259 | PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 260 | } |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 261 | |
| 262 | static inline void write_gdt_entry(struct desc_struct *dt, int entry, |
| 263 | void *desc, int type) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 264 | { |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 265 | PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 266 | } |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 267 | |
Glauber de Oliveira Costa | 8d94734 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 268 | static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 269 | { |
Glauber de Oliveira Costa | 8d94734 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 270 | PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 271 | } |
| 272 | static inline void set_iopl_mask(unsigned mask) |
| 273 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 274 | PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 275 | } |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 276 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 277 | /* The paravirtualized I/O functions */ |
Joe Perches | 49cd740 | 2008-03-23 01:03:00 -0700 | [diff] [blame] | 278 | static inline void slow_down_io(void) |
| 279 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 280 | pv_cpu_ops.io_delay(); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 281 | #ifdef REALLY_SLOW_IO |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 282 | pv_cpu_ops.io_delay(); |
| 283 | pv_cpu_ops.io_delay(); |
| 284 | pv_cpu_ops.io_delay(); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 285 | #endif |
| 286 | } |
| 287 | |
Zachary Amsden | ae5da27 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 288 | #ifdef CONFIG_SMP |
| 289 | static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, |
| 290 | unsigned long start_esp) |
| 291 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 292 | PVOP_VCALL3(pv_apic_ops.startup_ipi_hook, |
| 293 | phys_apicid, start_eip, start_esp); |
Zachary Amsden | ae5da27 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 294 | } |
| 295 | #endif |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 296 | |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 297 | static inline void paravirt_activate_mm(struct mm_struct *prev, |
| 298 | struct mm_struct *next) |
| 299 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 300 | PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 301 | } |
| 302 | |
Dave Hansen | a1ea1c0 | 2014-11-18 10:23:49 -0800 | [diff] [blame] | 303 | static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, |
| 304 | struct mm_struct *mm) |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 305 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 306 | PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 307 | } |
| 308 | |
Dave Hansen | a1ea1c0 | 2014-11-18 10:23:49 -0800 | [diff] [blame] | 309 | static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 310 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 311 | PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 312 | } |
| 313 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 314 | static inline void __flush_tlb(void) |
| 315 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 316 | PVOP_VCALL0(pv_mmu_ops.flush_tlb_user); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 317 | } |
| 318 | static inline void __flush_tlb_global(void) |
| 319 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 320 | PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 321 | } |
| 322 | static inline void __flush_tlb_single(unsigned long addr) |
| 323 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 324 | PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 325 | } |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 326 | |
Rusty Russell | 4595f96 | 2009-01-10 21:58:09 -0800 | [diff] [blame] | 327 | static inline void flush_tlb_others(const struct cpumask *cpumask, |
| 328 | struct mm_struct *mm, |
Alex Shi | e7b52ff | 2012-06-28 09:02:17 +0800 | [diff] [blame] | 329 | unsigned long start, |
| 330 | unsigned long end) |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 331 | { |
Alex Shi | e7b52ff | 2012-06-28 09:02:17 +0800 | [diff] [blame] | 332 | PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end); |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 333 | } |
| 334 | |
Jeremy Fitzhardinge | eba0045 | 2008-06-25 00:19:12 -0400 | [diff] [blame] | 335 | static inline int paravirt_pgd_alloc(struct mm_struct *mm) |
| 336 | { |
| 337 | return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm); |
| 338 | } |
| 339 | |
| 340 | static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) |
| 341 | { |
| 342 | PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd); |
| 343 | } |
| 344 | |
Eduardo Habkost | f863993 | 2008-07-30 18:32:27 -0300 | [diff] [blame] | 345 | static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 346 | { |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 347 | PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 348 | } |
Eduardo Habkost | f863993 | 2008-07-30 18:32:27 -0300 | [diff] [blame] | 349 | static inline void paravirt_release_pte(unsigned long pfn) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 350 | { |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 351 | PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 352 | } |
Zachary Amsden | c119ecc | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 353 | |
Eduardo Habkost | f863993 | 2008-07-30 18:32:27 -0300 | [diff] [blame] | 354 | static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 355 | { |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 356 | PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 357 | } |
| 358 | |
Eduardo Habkost | f863993 | 2008-07-30 18:32:27 -0300 | [diff] [blame] | 359 | static inline void paravirt_release_pmd(unsigned long pfn) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 360 | { |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 361 | PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 362 | } |
| 363 | |
Eduardo Habkost | f863993 | 2008-07-30 18:32:27 -0300 | [diff] [blame] | 364 | static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) |
Jeremy Fitzhardinge | 2761fa0 | 2008-03-17 16:37:02 -0700 | [diff] [blame] | 365 | { |
| 366 | PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn); |
| 367 | } |
Eduardo Habkost | f863993 | 2008-07-30 18:32:27 -0300 | [diff] [blame] | 368 | static inline void paravirt_release_pud(unsigned long pfn) |
Jeremy Fitzhardinge | 2761fa0 | 2008-03-17 16:37:02 -0700 | [diff] [blame] | 369 | { |
| 370 | PVOP_VCALL1(pv_mmu_ops.release_pud, pfn); |
| 371 | } |
| 372 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 373 | static inline void pte_update(struct mm_struct *mm, unsigned long addr, |
| 374 | pte_t *ptep) |
| 375 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 376 | PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 377 | } |
Andrea Arcangeli | 331127f | 2011-01-13 15:46:36 -0800 | [diff] [blame] | 378 | static inline void pmd_update(struct mm_struct *mm, unsigned long addr, |
| 379 | pmd_t *pmdp) |
| 380 | { |
| 381 | PVOP_VCALL3(pv_mmu_ops.pmd_update, mm, addr, pmdp); |
| 382 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 383 | |
| 384 | static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, |
| 385 | pte_t *ptep) |
| 386 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 387 | PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 388 | } |
| 389 | |
Andrea Arcangeli | 331127f | 2011-01-13 15:46:36 -0800 | [diff] [blame] | 390 | static inline void pmd_update_defer(struct mm_struct *mm, unsigned long addr, |
| 391 | pmd_t *pmdp) |
| 392 | { |
| 393 | PVOP_VCALL3(pv_mmu_ops.pmd_update_defer, mm, addr, pmdp); |
| 394 | } |
| 395 | |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 396 | static inline pte_t __pte(pteval_t val) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 397 | { |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 398 | pteval_t ret; |
| 399 | |
| 400 | if (sizeof(pteval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 401 | ret = PVOP_CALLEE2(pteval_t, |
| 402 | pv_mmu_ops.make_pte, |
| 403 | val, (u64)val >> 32); |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 404 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 405 | ret = PVOP_CALLEE1(pteval_t, |
| 406 | pv_mmu_ops.make_pte, |
| 407 | val); |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 408 | |
Jeremy Fitzhardinge | c8e5393 | 2008-01-30 13:32:57 +0100 | [diff] [blame] | 409 | return (pte_t) { .pte = ret }; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 410 | } |
| 411 | |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 412 | static inline pteval_t pte_val(pte_t pte) |
| 413 | { |
| 414 | pteval_t ret; |
| 415 | |
| 416 | if (sizeof(pteval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 417 | ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val, |
| 418 | pte.pte, (u64)pte.pte >> 32); |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 419 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 420 | ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val, |
| 421 | pte.pte); |
Jeremy Fitzhardinge | 773221f | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 422 | |
| 423 | return ret; |
| 424 | } |
| 425 | |
Jeremy Fitzhardinge | ef38503 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 426 | static inline pgd_t __pgd(pgdval_t val) |
| 427 | { |
| 428 | pgdval_t ret; |
| 429 | |
| 430 | if (sizeof(pgdval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 431 | ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd, |
| 432 | val, (u64)val >> 32); |
Jeremy Fitzhardinge | ef38503 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 433 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 434 | ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd, |
| 435 | val); |
Jeremy Fitzhardinge | ef38503 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 436 | |
| 437 | return (pgd_t) { ret }; |
| 438 | } |
| 439 | |
| 440 | static inline pgdval_t pgd_val(pgd_t pgd) |
| 441 | { |
| 442 | pgdval_t ret; |
| 443 | |
| 444 | if (sizeof(pgdval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 445 | ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val, |
| 446 | pgd.pgd, (u64)pgd.pgd >> 32); |
Jeremy Fitzhardinge | ef38503 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 447 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 448 | ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val, |
| 449 | pgd.pgd); |
Jeremy Fitzhardinge | ef38503 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 450 | |
| 451 | return ret; |
| 452 | } |
| 453 | |
Jeremy Fitzhardinge | 08b882c | 2008-06-16 04:30:01 -0700 | [diff] [blame] | 454 | #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION |
| 455 | static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, |
| 456 | pte_t *ptep) |
| 457 | { |
| 458 | pteval_t ret; |
| 459 | |
| 460 | ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start, |
| 461 | mm, addr, ptep); |
| 462 | |
| 463 | return (pte_t) { .pte = ret }; |
| 464 | } |
| 465 | |
| 466 | static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, |
| 467 | pte_t *ptep, pte_t pte) |
| 468 | { |
| 469 | if (sizeof(pteval_t) > sizeof(long)) |
| 470 | /* 5 arg words */ |
| 471 | pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte); |
| 472 | else |
| 473 | PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit, |
| 474 | mm, addr, ptep, pte.pte); |
| 475 | } |
| 476 | |
Jeremy Fitzhardinge | 4eed80c | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 477 | static inline void set_pte(pte_t *ptep, pte_t pte) |
| 478 | { |
| 479 | if (sizeof(pteval_t) > sizeof(long)) |
| 480 | PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, |
| 481 | pte.pte, (u64)pte.pte >> 32); |
| 482 | else |
| 483 | PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, |
| 484 | pte.pte); |
| 485 | } |
| 486 | |
| 487 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 488 | pte_t *ptep, pte_t pte) |
| 489 | { |
| 490 | if (sizeof(pteval_t) > sizeof(long)) |
| 491 | /* 5 arg words */ |
| 492 | pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); |
| 493 | else |
| 494 | PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); |
| 495 | } |
| 496 | |
Andrea Arcangeli | 331127f | 2011-01-13 15:46:36 -0800 | [diff] [blame] | 497 | static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, |
| 498 | pmd_t *pmdp, pmd_t pmd) |
| 499 | { |
Andrea Arcangeli | 331127f | 2011-01-13 15:46:36 -0800 | [diff] [blame] | 500 | if (sizeof(pmdval_t) > sizeof(long)) |
| 501 | /* 5 arg words */ |
| 502 | pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd); |
| 503 | else |
Andrea Arcangeli | cacf061 | 2011-01-25 15:07:09 -0800 | [diff] [blame] | 504 | PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp, |
| 505 | native_pmd_val(pmd)); |
Andrea Arcangeli | 331127f | 2011-01-13 15:46:36 -0800 | [diff] [blame] | 506 | } |
Andrea Arcangeli | 331127f | 2011-01-13 15:46:36 -0800 | [diff] [blame] | 507 | |
Jeremy Fitzhardinge | 60b3f62 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 508 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) |
| 509 | { |
| 510 | pmdval_t val = native_pmd_val(pmd); |
| 511 | |
| 512 | if (sizeof(pmdval_t) > sizeof(long)) |
| 513 | PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); |
| 514 | else |
| 515 | PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); |
| 516 | } |
| 517 | |
Kirill A. Shutemov | 9823336 | 2015-04-14 15:46:14 -0700 | [diff] [blame] | 518 | #if CONFIG_PGTABLE_LEVELS >= 3 |
Glauber de Oliveira Costa | 1fe9151 | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 519 | static inline pmd_t __pmd(pmdval_t val) |
| 520 | { |
| 521 | pmdval_t ret; |
| 522 | |
| 523 | if (sizeof(pmdval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 524 | ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd, |
| 525 | val, (u64)val >> 32); |
Glauber de Oliveira Costa | 1fe9151 | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 526 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 527 | ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd, |
| 528 | val); |
Glauber de Oliveira Costa | 1fe9151 | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 529 | |
| 530 | return (pmd_t) { ret }; |
| 531 | } |
| 532 | |
| 533 | static inline pmdval_t pmd_val(pmd_t pmd) |
| 534 | { |
| 535 | pmdval_t ret; |
| 536 | |
| 537 | if (sizeof(pmdval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 538 | ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val, |
| 539 | pmd.pmd, (u64)pmd.pmd >> 32); |
Glauber de Oliveira Costa | 1fe9151 | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 540 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 541 | ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val, |
| 542 | pmd.pmd); |
Glauber de Oliveira Costa | 1fe9151 | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 543 | |
| 544 | return ret; |
| 545 | } |
| 546 | |
| 547 | static inline void set_pud(pud_t *pudp, pud_t pud) |
| 548 | { |
| 549 | pudval_t val = native_pud_val(pud); |
| 550 | |
| 551 | if (sizeof(pudval_t) > sizeof(long)) |
| 552 | PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, |
| 553 | val, (u64)val >> 32); |
| 554 | else |
| 555 | PVOP_VCALL2(pv_mmu_ops.set_pud, pudp, |
| 556 | val); |
| 557 | } |
Kirill A. Shutemov | 9823336 | 2015-04-14 15:46:14 -0700 | [diff] [blame] | 558 | #if CONFIG_PGTABLE_LEVELS == 4 |
Eduardo Habkost | 9042219 | 2008-01-30 13:33:20 +0100 | [diff] [blame] | 559 | static inline pud_t __pud(pudval_t val) |
| 560 | { |
| 561 | pudval_t ret; |
| 562 | |
| 563 | if (sizeof(pudval_t) > sizeof(long)) |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 564 | ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud, |
| 565 | val, (u64)val >> 32); |
Eduardo Habkost | 9042219 | 2008-01-30 13:33:20 +0100 | [diff] [blame] | 566 | else |
Jeremy Fitzhardinge | da5de7c | 2009-01-28 14:35:07 -0800 | [diff] [blame] | 567 | ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud, |
| 568 | val); |
Eduardo Habkost | 9042219 | 2008-01-30 13:33:20 +0100 | [diff] [blame] | 569 | |
| 570 | return (pud_t) { ret }; |
| 571 | } |
| 572 | |
| 573 | static inline pudval_t pud_val(pud_t pud) |
| 574 | { |
| 575 | pudval_t ret; |
| 576 | |
| 577 | if (sizeof(pudval_t) > sizeof(long)) |
Jeremy Fitzhardinge | 4767afb | 2009-01-29 01:51:34 -0800 | [diff] [blame] | 578 | ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val, |
| 579 | pud.pud, (u64)pud.pud >> 32); |
Eduardo Habkost | 9042219 | 2008-01-30 13:33:20 +0100 | [diff] [blame] | 580 | else |
Jeremy Fitzhardinge | 4767afb | 2009-01-29 01:51:34 -0800 | [diff] [blame] | 581 | ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val, |
| 582 | pud.pud); |
Eduardo Habkost | 9042219 | 2008-01-30 13:33:20 +0100 | [diff] [blame] | 583 | |
| 584 | return ret; |
| 585 | } |
| 586 | |
| 587 | static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) |
| 588 | { |
| 589 | pgdval_t val = native_pgd_val(pgd); |
| 590 | |
| 591 | if (sizeof(pgdval_t) > sizeof(long)) |
| 592 | PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp, |
| 593 | val, (u64)val >> 32); |
| 594 | else |
| 595 | PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, |
| 596 | val); |
| 597 | } |
| 598 | |
| 599 | static inline void pgd_clear(pgd_t *pgdp) |
| 600 | { |
| 601 | set_pgd(pgdp, __pgd(0)); |
| 602 | } |
| 603 | |
| 604 | static inline void pud_clear(pud_t *pudp) |
| 605 | { |
| 606 | set_pud(pudp, __pud(0)); |
| 607 | } |
| 608 | |
Kirill A. Shutemov | 9823336 | 2015-04-14 15:46:14 -0700 | [diff] [blame] | 609 | #endif /* CONFIG_PGTABLE_LEVELS == 4 */ |
Eduardo Habkost | 9042219 | 2008-01-30 13:33:20 +0100 | [diff] [blame] | 610 | |
Kirill A. Shutemov | 9823336 | 2015-04-14 15:46:14 -0700 | [diff] [blame] | 611 | #endif /* CONFIG_PGTABLE_LEVELS >= 3 */ |
Glauber de Oliveira Costa | 1fe9151 | 2008-01-30 13:33:19 +0100 | [diff] [blame] | 612 | |
Jeremy Fitzhardinge | 4eed80c | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 613 | #ifdef CONFIG_X86_PAE |
| 614 | /* Special-case pte-setting operations for PAE, which can't update a |
| 615 | 64-bit pte atomically */ |
| 616 | static inline void set_pte_atomic(pte_t *ptep, pte_t pte) |
| 617 | { |
| 618 | PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, |
| 619 | pte.pte, pte.pte >> 32); |
| 620 | } |
| 621 | |
Jeremy Fitzhardinge | 4eed80c | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 622 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, |
| 623 | pte_t *ptep) |
| 624 | { |
| 625 | PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); |
| 626 | } |
Jeremy Fitzhardinge | 60b3f62 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 627 | |
| 628 | static inline void pmd_clear(pmd_t *pmdp) |
| 629 | { |
| 630 | PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); |
| 631 | } |
Jeremy Fitzhardinge | 4eed80c | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 632 | #else /* !CONFIG_X86_PAE */ |
| 633 | static inline void set_pte_atomic(pte_t *ptep, pte_t pte) |
| 634 | { |
| 635 | set_pte(ptep, pte); |
| 636 | } |
| 637 | |
Jeremy Fitzhardinge | 4eed80c | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 638 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, |
| 639 | pte_t *ptep) |
| 640 | { |
| 641 | set_pte_at(mm, addr, ptep, __pte(0)); |
| 642 | } |
Jeremy Fitzhardinge | 60b3f62 | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 643 | |
| 644 | static inline void pmd_clear(pmd_t *pmdp) |
| 645 | { |
| 646 | set_pmd(pmdp, __pmd(0)); |
| 647 | } |
Jeremy Fitzhardinge | 4eed80c | 2008-01-30 13:33:15 +0100 | [diff] [blame] | 648 | #endif /* CONFIG_X86_PAE */ |
| 649 | |
Jeremy Fitzhardinge | 7fd7d83 | 2009-02-17 23:24:03 -0800 | [diff] [blame] | 650 | #define __HAVE_ARCH_START_CONTEXT_SWITCH |
Jeremy Fitzhardinge | 224101e | 2009-02-18 11:18:57 -0800 | [diff] [blame] | 651 | static inline void arch_start_context_switch(struct task_struct *prev) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 652 | { |
Jeremy Fitzhardinge | 224101e | 2009-02-18 11:18:57 -0800 | [diff] [blame] | 653 | PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 654 | } |
| 655 | |
Jeremy Fitzhardinge | 224101e | 2009-02-18 11:18:57 -0800 | [diff] [blame] | 656 | static inline void arch_end_context_switch(struct task_struct *next) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 657 | { |
Jeremy Fitzhardinge | 224101e | 2009-02-18 11:18:57 -0800 | [diff] [blame] | 658 | PVOP_VCALL1(pv_cpu_ops.end_context_switch, next); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 659 | } |
| 660 | |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 661 | #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 662 | static inline void arch_enter_lazy_mmu_mode(void) |
| 663 | { |
Jeremy Fitzhardinge | 8965c1c0 | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 664 | PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 665 | } |
| 666 | |
| 667 | static inline void arch_leave_lazy_mmu_mode(void) |
| 668 | { |
Jeremy Fitzhardinge | 8965c1c0 | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 669 | PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 670 | } |
| 671 | |
Boris Ostrovsky | 511ba86 | 2013-03-23 09:36:36 -0400 | [diff] [blame] | 672 | static inline void arch_flush_lazy_mmu_mode(void) |
| 673 | { |
| 674 | PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush); |
| 675 | } |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 676 | |
Jeremy Fitzhardinge | aeaaa59 | 2008-06-17 11:42:01 -0700 | [diff] [blame] | 677 | static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, |
Masami Hiramatsu | 3b3809a | 2009-04-09 10:55:33 -0700 | [diff] [blame] | 678 | phys_addr_t phys, pgprot_t flags) |
Jeremy Fitzhardinge | aeaaa59 | 2008-06-17 11:42:01 -0700 | [diff] [blame] | 679 | { |
| 680 | pv_mmu_ops.set_fixmap(idx, phys, flags); |
| 681 | } |
| 682 | |
Jeremy Fitzhardinge | b4ecc12 | 2009-05-13 17:16:55 -0700 | [diff] [blame] | 683 | #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) |
Ingo Molnar | 4bb689e | 2008-07-09 14:33:33 +0200 | [diff] [blame] | 684 | |
Ingo Molnar | 62c7a1e | 2015-05-11 09:47:23 +0200 | [diff] [blame] | 685 | #ifdef CONFIG_QUEUED_SPINLOCKS |
Peter Zijlstra (Intel) | f233f7f | 2015-04-24 14:56:38 -0400 | [diff] [blame] | 686 | |
| 687 | static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, |
| 688 | u32 val) |
| 689 | { |
| 690 | PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val); |
| 691 | } |
| 692 | |
| 693 | static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) |
| 694 | { |
| 695 | PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock); |
| 696 | } |
| 697 | |
| 698 | static __always_inline void pv_wait(u8 *ptr, u8 val) |
| 699 | { |
| 700 | PVOP_VCALL2(pv_lock_ops.wait, ptr, val); |
| 701 | } |
| 702 | |
| 703 | static __always_inline void pv_kick(int cpu) |
| 704 | { |
| 705 | PVOP_VCALL1(pv_lock_ops.kick, cpu); |
| 706 | } |
| 707 | |
Ingo Molnar | 62c7a1e | 2015-05-11 09:47:23 +0200 | [diff] [blame] | 708 | #else /* !CONFIG_QUEUED_SPINLOCKS */ |
Peter Zijlstra (Intel) | f233f7f | 2015-04-24 14:56:38 -0400 | [diff] [blame] | 709 | |
Jeremy Fitzhardinge | 545ac13 | 2013-08-09 19:51:49 +0530 | [diff] [blame] | 710 | static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock, |
| 711 | __ticket_t ticket) |
Jeremy Fitzhardinge | 74d4aff | 2008-07-07 12:07:50 -0700 | [diff] [blame] | 712 | { |
Jeremy Fitzhardinge | 354714d | 2013-08-09 19:51:55 +0530 | [diff] [blame] | 713 | PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket); |
Jeremy Fitzhardinge | 74d4aff | 2008-07-07 12:07:50 -0700 | [diff] [blame] | 714 | } |
| 715 | |
Jeremy Fitzhardinge | 96f853e | 2013-08-09 19:51:58 +0530 | [diff] [blame] | 716 | static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock, |
Jeremy Fitzhardinge | 545ac13 | 2013-08-09 19:51:49 +0530 | [diff] [blame] | 717 | __ticket_t ticket) |
Jeremy Fitzhardinge | 74d4aff | 2008-07-07 12:07:50 -0700 | [diff] [blame] | 718 | { |
Jeremy Fitzhardinge | 545ac13 | 2013-08-09 19:51:49 +0530 | [diff] [blame] | 719 | PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket); |
Jeremy Fitzhardinge | 74d4aff | 2008-07-07 12:07:50 -0700 | [diff] [blame] | 720 | } |
| 721 | |
Ingo Molnar | 62c7a1e | 2015-05-11 09:47:23 +0200 | [diff] [blame] | 722 | #endif /* CONFIG_QUEUED_SPINLOCKS */ |
Peter Zijlstra (Intel) | f233f7f | 2015-04-24 14:56:38 -0400 | [diff] [blame] | 723 | |
| 724 | #endif /* SMP && PARAVIRT_SPINLOCKS */ |
Ingo Molnar | 4bb689e | 2008-07-09 14:33:33 +0200 | [diff] [blame] | 725 | |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 726 | #ifdef CONFIG_X86_32 |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 727 | #define PV_SAVE_REGS "pushl %ecx; pushl %edx;" |
| 728 | #define PV_RESTORE_REGS "popl %edx; popl %ecx;" |
| 729 | |
| 730 | /* save and restore all caller-save registers, except return value */ |
Jeremy Fitzhardinge | e584f55 | 2009-01-30 23:17:23 -0800 | [diff] [blame] | 731 | #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" |
| 732 | #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 733 | |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 734 | #define PV_FLAGS_ARG "0" |
| 735 | #define PV_EXTRA_CLOBBERS |
| 736 | #define PV_VEXTRA_CLOBBERS |
| 737 | #else |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 738 | /* save and restore all caller-save registers, except return value */ |
| 739 | #define PV_SAVE_ALL_CALLER_REGS \ |
| 740 | "push %rcx;" \ |
| 741 | "push %rdx;" \ |
| 742 | "push %rsi;" \ |
| 743 | "push %rdi;" \ |
| 744 | "push %r8;" \ |
| 745 | "push %r9;" \ |
| 746 | "push %r10;" \ |
| 747 | "push %r11;" |
| 748 | #define PV_RESTORE_ALL_CALLER_REGS \ |
| 749 | "pop %r11;" \ |
| 750 | "pop %r10;" \ |
| 751 | "pop %r9;" \ |
| 752 | "pop %r8;" \ |
| 753 | "pop %rdi;" \ |
| 754 | "pop %rsi;" \ |
| 755 | "pop %rdx;" \ |
| 756 | "pop %rcx;" |
| 757 | |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 758 | /* We save some registers, but all of them, that's too much. We clobber all |
| 759 | * caller saved registers but the argument parameter */ |
| 760 | #define PV_SAVE_REGS "pushq %%rdi;" |
| 761 | #define PV_RESTORE_REGS "popq %%rdi;" |
Jeremy Fitzhardinge | c24481e | 2008-07-08 15:07:12 -0700 | [diff] [blame] | 762 | #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi" |
| 763 | #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi" |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 764 | #define PV_FLAGS_ARG "D" |
| 765 | #endif |
| 766 | |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 767 | /* |
| 768 | * Generate a thunk around a function which saves all caller-save |
| 769 | * registers except for the return value. This allows C functions to |
| 770 | * be called from assembler code where fewer than normal registers are |
| 771 | * available. It may also help code generation around calls from C |
| 772 | * code if the common case doesn't use many registers. |
| 773 | * |
| 774 | * When a callee is wrapped in a thunk, the caller can assume that all |
| 775 | * arg regs and all scratch registers are preserved across the |
| 776 | * call. The return value in rax/eax will not be saved, even for void |
| 777 | * functions. |
| 778 | */ |
| 779 | #define PV_CALLEE_SAVE_REGS_THUNK(func) \ |
| 780 | extern typeof(func) __raw_callee_save_##func; \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 781 | \ |
| 782 | asm(".pushsection .text;" \ |
Andi Kleen | a2e7f0e | 2013-10-22 09:07:56 -0700 | [diff] [blame] | 783 | ".globl __raw_callee_save_" #func " ; " \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 784 | "__raw_callee_save_" #func ": " \ |
| 785 | PV_SAVE_ALL_CALLER_REGS \ |
| 786 | "call " #func ";" \ |
| 787 | PV_RESTORE_ALL_CALLER_REGS \ |
| 788 | "ret;" \ |
| 789 | ".popsection") |
| 790 | |
| 791 | /* Get a reference to a callee-save function */ |
| 792 | #define PV_CALLEE_SAVE(func) \ |
| 793 | ((struct paravirt_callee_save) { __raw_callee_save_##func }) |
| 794 | |
| 795 | /* Promise that "func" already uses the right calling convention */ |
| 796 | #define __PV_IS_CALLEE_SAVE(func) \ |
| 797 | ((struct paravirt_callee_save) { func }) |
| 798 | |
Steven Rostedt | b590854 | 2010-11-10 22:29:49 -0500 | [diff] [blame] | 799 | static inline notrace unsigned long arch_local_save_flags(void) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 800 | { |
Jeremy Fitzhardinge | 71999d9 | 2009-10-12 16:32:43 -0700 | [diff] [blame] | 801 | return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 802 | } |
| 803 | |
Steven Rostedt | b590854 | 2010-11-10 22:29:49 -0500 | [diff] [blame] | 804 | static inline notrace void arch_local_irq_restore(unsigned long f) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 805 | { |
Jeremy Fitzhardinge | 71999d9 | 2009-10-12 16:32:43 -0700 | [diff] [blame] | 806 | PVOP_VCALLEE1(pv_irq_ops.restore_fl, f); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 807 | } |
| 808 | |
Steven Rostedt | b590854 | 2010-11-10 22:29:49 -0500 | [diff] [blame] | 809 | static inline notrace void arch_local_irq_disable(void) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 810 | { |
Jeremy Fitzhardinge | 71999d9 | 2009-10-12 16:32:43 -0700 | [diff] [blame] | 811 | PVOP_VCALLEE0(pv_irq_ops.irq_disable); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 812 | } |
| 813 | |
Steven Rostedt | b590854 | 2010-11-10 22:29:49 -0500 | [diff] [blame] | 814 | static inline notrace void arch_local_irq_enable(void) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 815 | { |
Jeremy Fitzhardinge | 71999d9 | 2009-10-12 16:32:43 -0700 | [diff] [blame] | 816 | PVOP_VCALLEE0(pv_irq_ops.irq_enable); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 817 | } |
| 818 | |
Steven Rostedt | b590854 | 2010-11-10 22:29:49 -0500 | [diff] [blame] | 819 | static inline notrace unsigned long arch_local_irq_save(void) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 820 | { |
| 821 | unsigned long f; |
| 822 | |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 823 | f = arch_local_save_flags(); |
| 824 | arch_local_irq_disable(); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 825 | return f; |
| 826 | } |
| 827 | |
Jeremy Fitzhardinge | 74d4aff | 2008-07-07 12:07:50 -0700 | [diff] [blame] | 828 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 829 | /* Make sure as little as possible of this mess escapes. */ |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 830 | #undef PARAVIRT_CALL |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 831 | #undef __PVOP_CALL |
| 832 | #undef __PVOP_VCALL |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 833 | #undef PVOP_VCALL0 |
| 834 | #undef PVOP_CALL0 |
| 835 | #undef PVOP_VCALL1 |
| 836 | #undef PVOP_CALL1 |
| 837 | #undef PVOP_VCALL2 |
| 838 | #undef PVOP_CALL2 |
| 839 | #undef PVOP_VCALL3 |
| 840 | #undef PVOP_CALL3 |
| 841 | #undef PVOP_VCALL4 |
| 842 | #undef PVOP_CALL4 |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 843 | |
Thomas Gleixner | 6f30c1a | 2009-08-20 13:19:57 +0200 | [diff] [blame] | 844 | extern void default_banner(void); |
| 845 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 846 | #else /* __ASSEMBLY__ */ |
| 847 | |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 848 | #define _PVSITE(ptype, clobbers, ops, word, algn) \ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 849 | 771:; \ |
| 850 | ops; \ |
| 851 | 772:; \ |
| 852 | .pushsection .parainstructions,"a"; \ |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 853 | .align algn; \ |
| 854 | word 771b; \ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 855 | .byte ptype; \ |
| 856 | .byte 772b-771b; \ |
| 857 | .short clobbers; \ |
| 858 | .popsection |
| 859 | |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 860 | |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 861 | #define COND_PUSH(set, mask, reg) \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 862 | .if ((~(set)) & mask); push %reg; .endif |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 863 | #define COND_POP(set, mask, reg) \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 864 | .if ((~(set)) & mask); pop %reg; .endif |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 865 | |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 866 | #ifdef CONFIG_X86_64 |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 867 | |
| 868 | #define PV_SAVE_REGS(set) \ |
| 869 | COND_PUSH(set, CLBR_RAX, rax); \ |
| 870 | COND_PUSH(set, CLBR_RCX, rcx); \ |
| 871 | COND_PUSH(set, CLBR_RDX, rdx); \ |
| 872 | COND_PUSH(set, CLBR_RSI, rsi); \ |
| 873 | COND_PUSH(set, CLBR_RDI, rdi); \ |
| 874 | COND_PUSH(set, CLBR_R8, r8); \ |
| 875 | COND_PUSH(set, CLBR_R9, r9); \ |
| 876 | COND_PUSH(set, CLBR_R10, r10); \ |
| 877 | COND_PUSH(set, CLBR_R11, r11) |
| 878 | #define PV_RESTORE_REGS(set) \ |
| 879 | COND_POP(set, CLBR_R11, r11); \ |
| 880 | COND_POP(set, CLBR_R10, r10); \ |
| 881 | COND_POP(set, CLBR_R9, r9); \ |
| 882 | COND_POP(set, CLBR_R8, r8); \ |
| 883 | COND_POP(set, CLBR_RDI, rdi); \ |
| 884 | COND_POP(set, CLBR_RSI, rsi); \ |
| 885 | COND_POP(set, CLBR_RDX, rdx); \ |
| 886 | COND_POP(set, CLBR_RCX, rcx); \ |
| 887 | COND_POP(set, CLBR_RAX, rax) |
| 888 | |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 889 | #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8) |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 890 | #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8) |
Jeremy Fitzhardinge | 491eccb | 2008-06-25 00:19:15 -0400 | [diff] [blame] | 891 | #define PARA_INDIRECT(addr) *addr(%rip) |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 892 | #else |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 893 | #define PV_SAVE_REGS(set) \ |
| 894 | COND_PUSH(set, CLBR_EAX, eax); \ |
| 895 | COND_PUSH(set, CLBR_EDI, edi); \ |
| 896 | COND_PUSH(set, CLBR_ECX, ecx); \ |
| 897 | COND_PUSH(set, CLBR_EDX, edx) |
| 898 | #define PV_RESTORE_REGS(set) \ |
| 899 | COND_POP(set, CLBR_EDX, edx); \ |
| 900 | COND_POP(set, CLBR_ECX, ecx); \ |
| 901 | COND_POP(set, CLBR_EDI, edi); \ |
| 902 | COND_POP(set, CLBR_EAX, eax) |
| 903 | |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 904 | #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 905 | #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) |
Jeremy Fitzhardinge | 491eccb | 2008-06-25 00:19:15 -0400 | [diff] [blame] | 906 | #define PARA_INDIRECT(addr) *%cs:addr |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 907 | #endif |
| 908 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 909 | #define INTERRUPT_RETURN \ |
| 910 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ |
Jeremy Fitzhardinge | 491eccb | 2008-06-25 00:19:15 -0400 | [diff] [blame] | 911 | jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret)) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 912 | |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 913 | #define DISABLE_INTERRUPTS(clobbers) \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 914 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 915 | PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ |
Jeremy Fitzhardinge | 491eccb | 2008-06-25 00:19:15 -0400 | [diff] [blame] | 916 | call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 917 | PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 918 | |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 919 | #define ENABLE_INTERRUPTS(clobbers) \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 920 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 921 | PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ |
Jeremy Fitzhardinge | 491eccb | 2008-06-25 00:19:15 -0400 | [diff] [blame] | 922 | call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \ |
Jeremy Fitzhardinge | ecb93d1 | 2009-01-28 14:35:05 -0800 | [diff] [blame] | 923 | PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 924 | |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 925 | #define USERGS_SYSRET32 \ |
| 926 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \ |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 927 | CLBR_NONE, \ |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 928 | jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32)) |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 929 | |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 930 | #ifdef CONFIG_X86_32 |
Jeremy Fitzhardinge | 491eccb | 2008-06-25 00:19:15 -0400 | [diff] [blame] | 931 | #define GET_CR0_INTO_EAX \ |
| 932 | push %ecx; push %edx; \ |
| 933 | call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \ |
Jeremy Fitzhardinge | 42c24fa | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 934 | pop %edx; pop %ecx |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 935 | |
| 936 | #define ENABLE_INTERRUPTS_SYSEXIT \ |
| 937 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \ |
| 938 | CLBR_NONE, \ |
| 939 | jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit)) |
| 940 | |
| 941 | |
| 942 | #else /* !CONFIG_X86_32 */ |
Jeremy Fitzhardinge | a00394f | 2008-06-25 00:19:30 -0400 | [diff] [blame] | 943 | |
| 944 | /* |
| 945 | * If swapgs is used while the userspace stack is still current, |
| 946 | * there's no way to call a pvop. The PV replacement *must* be |
| 947 | * inlined, or the swapgs instruction must be trapped and emulated. |
| 948 | */ |
| 949 | #define SWAPGS_UNSAFE_STACK \ |
| 950 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ |
| 951 | swapgs) |
| 952 | |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 953 | /* |
| 954 | * Note: swapgs is very special, and in practise is either going to be |
| 955 | * implemented with a single "swapgs" instruction or something very |
| 956 | * special. Either way, we don't need to save any registers for |
| 957 | * it. |
| 958 | */ |
Glauber de Oliveira Costa | e801f86 | 2008-01-30 13:32:08 +0100 | [diff] [blame] | 959 | #define SWAPGS \ |
| 960 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ |
Jeremy Fitzhardinge | 9104a18 | 2009-01-28 14:35:04 -0800 | [diff] [blame] | 961 | call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \ |
Glauber de Oliveira Costa | e801f86 | 2008-01-30 13:32:08 +0100 | [diff] [blame] | 962 | ) |
| 963 | |
H. Peter Anvin | ffc4bc9 | 2012-04-18 17:16:48 -0700 | [diff] [blame] | 964 | #define GET_CR2_INTO_RAX \ |
| 965 | call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2) |
Glauber de Oliveira Costa | 4a8c4c4 | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 966 | |
Jeremy Fitzhardinge | fab5842 | 2008-06-25 00:19:31 -0400 | [diff] [blame] | 967 | #define PARAVIRT_ADJUST_EXCEPTION_FRAME \ |
| 968 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \ |
| 969 | CLBR_NONE, \ |
| 970 | call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame)) |
| 971 | |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 972 | #define USERGS_SYSRET64 \ |
| 973 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \ |
Jeremy Fitzhardinge | d75cd22 | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 974 | CLBR_NONE, \ |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 975 | jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64)) |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 976 | #endif /* CONFIG_X86_32 */ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 977 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 978 | #endif /* __ASSEMBLY__ */ |
Thomas Gleixner | 6f30c1a | 2009-08-20 13:19:57 +0200 | [diff] [blame] | 979 | #else /* CONFIG_PARAVIRT */ |
| 980 | # define default_banner x86_init_noop |
Dave Hansen | a1ea1c0 | 2014-11-18 10:23:49 -0800 | [diff] [blame] | 981 | #ifndef __ASSEMBLY__ |
| 982 | static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, |
| 983 | struct mm_struct *mm) |
| 984 | { |
| 985 | } |
| 986 | |
| 987 | static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) |
| 988 | { |
| 989 | } |
| 990 | #endif /* __ASSEMBLY__ */ |
Thomas Gleixner | 6f30c1a | 2009-08-20 13:19:57 +0200 | [diff] [blame] | 991 | #endif /* !CONFIG_PARAVIRT */ |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 992 | #endif /* _ASM_X86_PARAVIRT_H */ |