Thomas Gleixner | caab277 | 2019-06-03 07:44:50 +0200 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2012,2013 - ARM Ltd |
| 4 | * Author: Marc Zyngier <marc.zyngier@arm.com> |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 5 | */ |
| 6 | |
| 7 | #ifndef __ARM_KVM_ASM_H__ |
| 8 | #define __ARM_KVM_ASM_H__ |
| 9 | |
David Brazdil | ce492a1 | 2020-09-22 21:49:02 +0100 | [diff] [blame] | 10 | #include <asm/hyp_image.h> |
Mark Rutland | 3e00e39 | 2021-06-09 11:23:01 +0100 | [diff] [blame] | 11 | #include <asm/insn.h> |
Marc Zyngier | 45451914 | 2013-06-26 15:16:40 +0100 | [diff] [blame] | 12 | #include <asm/virt.h> |
| 13 | |
Marc Zyngier | 2016340 | 2016-09-06 14:02:05 +0100 | [diff] [blame] | 14 | #define ARM_EXIT_WITH_SERROR_BIT 31 |
| 15 | #define ARM_EXCEPTION_CODE(x) ((x) & ~(1U << ARM_EXIT_WITH_SERROR_BIT)) |
Marc Zyngier | 5846676 | 2018-12-19 08:28:38 +0000 | [diff] [blame] | 16 | #define ARM_EXCEPTION_IS_TRAP(x) (ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_TRAP) |
Marc Zyngier | 2016340 | 2016-09-06 14:02:05 +0100 | [diff] [blame] | 17 | #define ARM_SERROR_PENDING(x) !!((x) & (1U << ARM_EXIT_WITH_SERROR_BIT)) |
| 18 | |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 19 | #define ARM_EXCEPTION_IRQ 0 |
Marc Zyngier | 9aecafc | 2016-09-06 14:02:02 +0100 | [diff] [blame] | 20 | #define ARM_EXCEPTION_EL1_SERROR 1 |
| 21 | #define ARM_EXCEPTION_TRAP 2 |
Christoffer Dall | e4e11cc | 2018-10-17 20:21:16 +0200 | [diff] [blame] | 22 | #define ARM_EXCEPTION_IL 3 |
James Morse | c94b0cf | 2016-04-27 17:47:04 +0100 | [diff] [blame] | 23 | /* The hyp-stub will return this for any kvm_call_hyp() call */ |
Marc Zyngier | 4993fdc | 2017-04-03 19:37:37 +0100 | [diff] [blame] | 24 | #define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 25 | |
Christoffer Dall | 71a7e47 | 2018-12-03 21:31:24 +0100 | [diff] [blame] | 26 | #define kvm_arm_exception_type \ |
| 27 | {ARM_EXCEPTION_IRQ, "IRQ" }, \ |
| 28 | {ARM_EXCEPTION_EL1_SERROR, "SERROR" }, \ |
| 29 | {ARM_EXCEPTION_TRAP, "TRAP" }, \ |
| 30 | {ARM_EXCEPTION_HYP_GONE, "HYP_GONE" } |
| 31 | |
James Morse | 3dbf100 | 2019-06-18 16:17:34 +0100 | [diff] [blame] | 32 | /* |
| 33 | * Size of the HYP vectors preamble. kvm_patch_vector_branch() generates code |
| 34 | * that jumps over this. |
| 35 | */ |
James Morse | 0e5b9c0 | 2019-06-18 16:17:36 +0100 | [diff] [blame] | 36 | #define KVM_VECTOR_PREAMBLE (2 * AARCH64_INSN_SIZE) |
James Morse | 3dbf100 | 2019-06-18 16:17:34 +0100 | [diff] [blame] | 37 | |
Andrew Scull | 0546983 | 2020-09-15 11:46:41 +0100 | [diff] [blame] | 38 | #define KVM_HOST_SMCCC_ID(id) \ |
| 39 | ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ |
| 40 | ARM_SMCCC_SMC_64, \ |
| 41 | ARM_SMCCC_OWNER_VENDOR_HYP, \ |
| 42 | (id)) |
| 43 | |
| 44 | #define KVM_HOST_SMCCC_FUNC(name) KVM_HOST_SMCCC_ID(__KVM_HOST_SMCCC_FUNC_##name) |
| 45 | |
| 46 | #define __KVM_HOST_SMCCC_FUNC___kvm_hyp_init 0 |
Andrew Scull | 0546983 | 2020-09-15 11:46:41 +0100 | [diff] [blame] | 47 | |
Mark Rutland | 46c4a30 | 2018-05-10 12:13:47 +0100 | [diff] [blame] | 48 | #ifndef __ASSEMBLY__ |
| 49 | |
| 50 | #include <linux/mm.h> |
Marc Zyngier | 0c557ed | 2014-04-24 10:24:46 +0100 | [diff] [blame] | 51 | |
Marc Zyngier | a78738e | 2021-10-08 14:58:34 +0100 | [diff] [blame] | 52 | enum __kvm_host_smccc_func { |
Will Deacon | 057bed2 | 2021-10-08 14:58:39 +0100 | [diff] [blame] | 53 | /* Hypercalls available only prior to pKVM finalisation */ |
Marc Zyngier | a78738e | 2021-10-08 14:58:34 +0100 | [diff] [blame] | 54 | /* __KVM_HOST_SMCCC_FUNC___kvm_hyp_init */ |
Will Deacon | 057bed2 | 2021-10-08 14:58:39 +0100 | [diff] [blame] | 55 | __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 = __KVM_HOST_SMCCC_FUNC___kvm_hyp_init + 1, |
| 56 | __KVM_HOST_SMCCC_FUNC___pkvm_init, |
| 57 | __KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping, |
| 58 | __KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector, |
| 59 | __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs, |
| 60 | __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, |
| 61 | __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, |
| 62 | __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize, |
| 63 | |
| 64 | /* Hypercalls available after pKVM finalisation */ |
| 65 | __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, |
Will Deacon | b8cc6eb5 | 2021-12-15 16:12:30 +0000 | [diff] [blame] | 66 | __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, |
Will Deacon | 057bed2 | 2021-10-08 14:58:39 +0100 | [diff] [blame] | 67 | __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc, |
| 68 | __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, |
Marc Zyngier | a78738e | 2021-10-08 14:58:34 +0100 | [diff] [blame] | 69 | __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, |
| 70 | __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, |
| 71 | __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, |
| 72 | __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, |
| 73 | __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, |
Marc Zyngier | a78738e | 2021-10-08 14:58:34 +0100 | [diff] [blame] | 74 | __KVM_HOST_SMCCC_FUNC___vgic_v3_read_vmcr, |
| 75 | __KVM_HOST_SMCCC_FUNC___vgic_v3_write_vmcr, |
Marc Zyngier | a78738e | 2021-10-08 14:58:34 +0100 | [diff] [blame] | 76 | __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, |
| 77 | __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs, |
Marc Zyngier | be08c3c | 2021-10-18 17:20:50 +0100 | [diff] [blame] | 78 | __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_init_traps, |
Fuad Tabba | a1ec5c7 | 2022-11-10 19:02:45 +0000 | [diff] [blame] | 79 | __KVM_HOST_SMCCC_FUNC___pkvm_init_vm, |
| 80 | __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu, |
| 81 | __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm, |
Marc Zyngier | a78738e | 2021-10-08 14:58:34 +0100 | [diff] [blame] | 82 | }; |
| 83 | |
Andrew Scull | f50b6f6a | 2020-06-25 14:14:10 +0100 | [diff] [blame] | 84 | #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] |
| 85 | #define DECLARE_KVM_NVHE_SYM(sym) extern char kvm_nvhe_sym(sym)[] |
| 86 | |
| 87 | /* |
| 88 | * Define a pair of symbols sharing the same name but one defined in |
| 89 | * VHE and the other in nVHE hyp implementations. |
| 90 | */ |
| 91 | #define DECLARE_KVM_HYP_SYM(sym) \ |
| 92 | DECLARE_KVM_VHE_SYM(sym); \ |
| 93 | DECLARE_KVM_NVHE_SYM(sym) |
| 94 | |
David Brazdil | 5724949 | 2020-09-22 21:49:06 +0100 | [diff] [blame] | 95 | #define DECLARE_KVM_VHE_PER_CPU(type, sym) \ |
| 96 | DECLARE_PER_CPU(type, sym) |
| 97 | #define DECLARE_KVM_NVHE_PER_CPU(type, sym) \ |
| 98 | DECLARE_PER_CPU(type, kvm_nvhe_sym(sym)) |
| 99 | |
| 100 | #define DECLARE_KVM_HYP_PER_CPU(type, sym) \ |
| 101 | DECLARE_KVM_VHE_PER_CPU(type, sym); \ |
| 102 | DECLARE_KVM_NVHE_PER_CPU(type, sym) |
| 103 | |
David Brazdil | 30c9539 | 2020-09-22 21:49:09 +0100 | [diff] [blame] | 104 | /* |
| 105 | * Compute pointer to a symbol defined in nVHE percpu region. |
| 106 | * Returns NULL if percpu memory has not been allocated yet. |
| 107 | */ |
| 108 | #define this_cpu_ptr_nvhe_sym(sym) per_cpu_ptr_nvhe_sym(sym, smp_processor_id()) |
| 109 | #define per_cpu_ptr_nvhe_sym(sym, cpu) \ |
| 110 | ({ \ |
| 111 | unsigned long base, off; \ |
Quentin Perret | fe41a7f | 2022-11-10 19:02:54 +0000 | [diff] [blame] | 112 | base = kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]; \ |
David Brazdil | 30c9539 | 2020-09-22 21:49:09 +0100 | [diff] [blame] | 113 | off = (unsigned long)&CHOOSE_NVHE_SYM(sym) - \ |
| 114 | (unsigned long)&CHOOSE_NVHE_SYM(__per_cpu_start); \ |
| 115 | base ? (typeof(CHOOSE_NVHE_SYM(sym))*)(base + off) : NULL; \ |
| 116 | }) |
David Brazdil | 5724949 | 2020-09-22 21:49:06 +0100 | [diff] [blame] | 117 | |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 118 | #if defined(__KVM_NVHE_HYPERVISOR__) |
Marc Zyngier | 6de7dd3 | 2020-07-07 15:34:06 +0100 | [diff] [blame] | 119 | |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 120 | #define CHOOSE_NVHE_SYM(sym) sym |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 121 | #define CHOOSE_HYP_SYM(sym) CHOOSE_NVHE_SYM(sym) |
| 122 | |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 123 | /* The nVHE hypervisor shouldn't even try to access VHE symbols */ |
| 124 | extern void *__nvhe_undefined_symbol; |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 125 | #define CHOOSE_VHE_SYM(sym) __nvhe_undefined_symbol |
| 126 | #define this_cpu_ptr_hyp_sym(sym) (&__nvhe_undefined_symbol) |
| 127 | #define per_cpu_ptr_hyp_sym(sym, cpu) (&__nvhe_undefined_symbol) |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 128 | |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 129 | #elif defined(__KVM_VHE_HYPERVISOR__) |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 130 | |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 131 | #define CHOOSE_VHE_SYM(sym) sym |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 132 | #define CHOOSE_HYP_SYM(sym) CHOOSE_VHE_SYM(sym) |
| 133 | |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 134 | /* The VHE hypervisor shouldn't even try to access nVHE symbols */ |
| 135 | extern void *__vhe_undefined_symbol; |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 136 | #define CHOOSE_NVHE_SYM(sym) __vhe_undefined_symbol |
| 137 | #define this_cpu_ptr_hyp_sym(sym) (&__vhe_undefined_symbol) |
| 138 | #define per_cpu_ptr_hyp_sym(sym, cpu) (&__vhe_undefined_symbol) |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 139 | |
| 140 | #else |
| 141 | |
Marc Zyngier | 6de7dd3 | 2020-07-07 15:34:06 +0100 | [diff] [blame] | 142 | /* |
| 143 | * BIG FAT WARNINGS: |
| 144 | * |
| 145 | * - Don't be tempted to change the following is_kernel_in_hyp_mode() |
| 146 | * to has_vhe(). has_vhe() is implemented as a *final* capability, |
| 147 | * while this is used early at boot time, when the capabilities are |
| 148 | * not final yet.... |
| 149 | * |
| 150 | * - Don't let the nVHE hypervisor have access to this, as it will |
| 151 | * pick the *wrong* symbol (yes, it runs at EL2...). |
| 152 | */ |
David Brazdil | 5724949 | 2020-09-22 21:49:06 +0100 | [diff] [blame] | 153 | #define CHOOSE_HYP_SYM(sym) (is_kernel_in_hyp_mode() \ |
| 154 | ? CHOOSE_VHE_SYM(sym) \ |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 155 | : CHOOSE_NVHE_SYM(sym)) |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 156 | |
David Brazdil | 5724949 | 2020-09-22 21:49:06 +0100 | [diff] [blame] | 157 | #define this_cpu_ptr_hyp_sym(sym) (is_kernel_in_hyp_mode() \ |
| 158 | ? this_cpu_ptr(&sym) \ |
| 159 | : this_cpu_ptr_nvhe_sym(sym)) |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 160 | |
David Brazdil | 5724949 | 2020-09-22 21:49:06 +0100 | [diff] [blame] | 161 | #define per_cpu_ptr_hyp_sym(sym, cpu) (is_kernel_in_hyp_mode() \ |
| 162 | ? per_cpu_ptr(&sym, cpu) \ |
| 163 | : per_cpu_ptr_nvhe_sym(sym, cpu)) |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 164 | |
Andrew Scull | ceee2fe | 2020-09-15 11:46:28 +0100 | [diff] [blame] | 165 | #define CHOOSE_VHE_SYM(sym) sym |
| 166 | #define CHOOSE_NVHE_SYM(sym) kvm_nvhe_sym(sym) |
| 167 | |
Marc Zyngier | 6de7dd3 | 2020-07-07 15:34:06 +0100 | [diff] [blame] | 168 | #endif |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 169 | |
David Brazdil | 63fec24 | 2020-12-02 18:41:06 +0000 | [diff] [blame] | 170 | struct kvm_nvhe_init_params { |
David Brazdil | d3e1086 | 2020-12-02 18:41:07 +0000 | [diff] [blame] | 171 | unsigned long mair_el2; |
| 172 | unsigned long tcr_el2; |
David Brazdil | 63fec24 | 2020-12-02 18:41:06 +0000 | [diff] [blame] | 173 | unsigned long tpidr_el2; |
| 174 | unsigned long stack_hyp_va; |
Kalesh Singh | ce33543 | 2022-04-20 14:42:54 -0700 | [diff] [blame] | 175 | unsigned long stack_pa; |
David Brazdil | 63fec24 | 2020-12-02 18:41:06 +0000 | [diff] [blame] | 176 | phys_addr_t pgd_pa; |
Quentin Perret | 734864c | 2021-03-19 10:01:29 +0000 | [diff] [blame] | 177 | unsigned long hcr_el2; |
| 178 | unsigned long vttbr; |
| 179 | unsigned long vtcr; |
David Brazdil | 63fec24 | 2020-12-02 18:41:06 +0000 | [diff] [blame] | 180 | }; |
| 181 | |
Kalesh Singh | 879e5ac | 2022-07-26 00:37:42 -0700 | [diff] [blame] | 182 | /* |
| 183 | * Used by the host in EL1 to dump the nVHE hypervisor backtrace on |
| 184 | * hyp_panic() in non-protected mode. |
| 185 | * |
| 186 | * @stack_base: hyp VA of the hyp_stack base. |
| 187 | * @overflow_stack_base: hyp VA of the hyp_overflow_stack base. |
| 188 | * @fp: hyp FP where the backtrace begins. |
| 189 | * @pc: hyp PC where the backtrace begins. |
| 190 | */ |
| 191 | struct kvm_nvhe_stacktrace_info { |
| 192 | unsigned long stack_base; |
| 193 | unsigned long overflow_stack_base; |
| 194 | unsigned long fp; |
| 195 | unsigned long pc; |
| 196 | }; |
| 197 | |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 198 | /* Translate a kernel address @ptr into its equivalent linear mapping */ |
| 199 | #define kvm_ksym_ref(ptr) \ |
Marc Zyngier | 2510ffe | 2016-03-18 17:25:59 +0000 | [diff] [blame] | 200 | ({ \ |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 201 | void *val = (ptr); \ |
Marc Zyngier | 2510ffe | 2016-03-18 17:25:59 +0000 | [diff] [blame] | 202 | if (!is_kernel_in_hyp_mode()) \ |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 203 | val = lm_alias((ptr)); \ |
Marc Zyngier | 2510ffe | 2016-03-18 17:25:59 +0000 | [diff] [blame] | 204 | val; \ |
| 205 | }) |
Andrew Scull | f50b6f6a | 2020-06-25 14:14:10 +0100 | [diff] [blame] | 206 | #define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(kvm_nvhe_sym(sym)) |
Ard Biesheuvel | a0bf977 | 2016-02-16 13:52:39 +0100 | [diff] [blame] | 207 | |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 208 | struct kvm; |
| 209 | struct kvm_vcpu; |
Christoffer Dall | a0e50aa | 2019-01-04 21:09:05 +0100 | [diff] [blame] | 210 | struct kvm_s2_mmu; |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 211 | |
Andrew Scull | 208243c | 2020-06-25 14:14:12 +0100 | [diff] [blame] | 212 | DECLARE_KVM_NVHE_SYM(__kvm_hyp_init); |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 213 | DECLARE_KVM_HYP_SYM(__kvm_hyp_vector); |
Andrew Scull | 208243c | 2020-06-25 14:14:12 +0100 | [diff] [blame] | 214 | #define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init) |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 215 | #define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector) |
| 216 | |
Quentin Perret | fe41a7f | 2022-11-10 19:02:54 +0000 | [diff] [blame] | 217 | extern unsigned long kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[]; |
David Brazdil | 30c9539 | 2020-09-22 21:49:09 +0100 | [diff] [blame] | 218 | DECLARE_KVM_NVHE_SYM(__per_cpu_start); |
| 219 | DECLARE_KVM_NVHE_SYM(__per_cpu_end); |
| 220 | |
David Brazdil | b877e98 | 2020-06-25 14:14:11 +0100 | [diff] [blame] | 221 | DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs); |
| 222 | #define __bp_harden_hyp_vecs CHOOSE_HYP_SYM(__bp_harden_hyp_vecs) |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 223 | |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 224 | extern void __kvm_flush_vm_context(void); |
Marc Zyngier | 01dc926 | 2021-03-03 16:45:05 +0000 | [diff] [blame] | 225 | extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu); |
Marc Zyngier | efaa5b9 | 2019-01-02 12:34:25 +0000 | [diff] [blame] | 226 | extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, |
| 227 | int level); |
Christoffer Dall | a0e50aa | 2019-01-04 21:09:05 +0100 | [diff] [blame] | 228 | extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 229 | |
Marc Zyngier | c6fe89f | 2020-05-13 11:58:29 +0100 | [diff] [blame] | 230 | extern void __kvm_timer_set_cntvoff(u64 cntvoff); |
Christoffer Dall | 688c50a | 2017-01-04 16:10:28 +0100 | [diff] [blame] | 231 | |
David Brazdil | 09cf57e | 2020-06-25 14:14:14 +0100 | [diff] [blame] | 232 | extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); |
Marc Zyngier | 1a9b130 | 2013-06-21 11:57:56 +0100 | [diff] [blame] | 233 | |
Marc Zyngier | f5e3068 | 2021-05-06 14:31:42 +0100 | [diff] [blame] | 234 | extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu); |
| 235 | |
Marc Zyngier | b9d699e | 2021-03-05 18:52:52 +0000 | [diff] [blame] | 236 | extern u64 __vgic_v3_get_gic_config(void); |
Christoffer Dall | 328e5664 | 2016-03-24 11:21:04 +0100 | [diff] [blame] | 237 | extern u64 __vgic_v3_read_vmcr(void); |
| 238 | extern void __vgic_v3_write_vmcr(u32 vmcr); |
Marc Zyngier | 0d98d00 | 2016-03-03 15:43:58 +0000 | [diff] [blame] | 239 | extern void __vgic_v3_init_lrs(void); |
Marc Zyngier | b2fb1c0 | 2013-07-12 15:15:23 +0100 | [diff] [blame] | 240 | |
Fuad Tabba | d6c850d | 2021-08-17 09:11:22 +0100 | [diff] [blame] | 241 | extern u64 __kvm_get_mdcr_el2(void); |
Alex Bennée | 56c7f5e | 2015-07-07 17:29:56 +0100 | [diff] [blame] | 242 | |
James Morse | 88a84cc | 2020-08-21 15:07:06 +0100 | [diff] [blame] | 243 | #define __KVM_EXTABLE(from, to) \ |
| 244 | " .pushsection __kvm_ex_table, \"a\"\n" \ |
| 245 | " .align 3\n" \ |
| 246 | " .long (" #from " - .), (" #to " - .)\n" \ |
| 247 | " .popsection\n" |
| 248 | |
| 249 | |
| 250 | #define __kvm_at(at_op, addr) \ |
| 251 | ( { \ |
| 252 | int __kvm_at_err = 0; \ |
| 253 | u64 spsr, elr; \ |
| 254 | asm volatile( \ |
| 255 | " mrs %1, spsr_el2\n" \ |
| 256 | " mrs %2, elr_el2\n" \ |
| 257 | "1: at "at_op", %3\n" \ |
| 258 | " isb\n" \ |
| 259 | " b 9f\n" \ |
| 260 | "2: msr spsr_el2, %1\n" \ |
| 261 | " msr elr_el2, %2\n" \ |
| 262 | " mov %w0, %4\n" \ |
| 263 | "9:\n" \ |
| 264 | __KVM_EXTABLE(1b, 2b) \ |
| 265 | : "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr) \ |
| 266 | : "r" (addr), "i" (-EFAULT)); \ |
| 267 | __kvm_at_err; \ |
| 268 | } ) |
| 269 | |
| 270 | |
Christoffer Dall | 4464e21 | 2017-10-08 17:01:56 +0200 | [diff] [blame] | 271 | #else /* __ASSEMBLY__ */ |
| 272 | |
Marc Zyngier | 85478ba | 2018-05-29 13:11:15 +0100 | [diff] [blame] | 273 | .macro get_host_ctxt reg, tmp |
David Brazdil | ea39102 | 2020-09-22 21:49:05 +0100 | [diff] [blame] | 274 | adr_this_cpu \reg, kvm_host_data, \tmp |
Andrew Murray | 630a168 | 2019-04-09 20:22:11 +0100 | [diff] [blame] | 275 | add \reg, \reg, #HOST_DATA_CONTEXT |
Marc Zyngier | 85478ba | 2018-05-29 13:11:15 +0100 | [diff] [blame] | 276 | .endm |
| 277 | |
Christoffer Dall | 4464e21 | 2017-10-08 17:01:56 +0200 | [diff] [blame] | 278 | .macro get_vcpu_ptr vcpu, ctxt |
| 279 | get_host_ctxt \ctxt, \vcpu |
| 280 | ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] |
Christoffer Dall | 4464e21 | 2017-10-08 17:01:56 +0200 | [diff] [blame] | 281 | .endm |
| 282 | |
Andrew Scull | 7db2153 | 2020-09-15 11:46:34 +0100 | [diff] [blame] | 283 | .macro get_loaded_vcpu vcpu, ctxt |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 284 | adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu |
Andrew Scull | 7db2153 | 2020-09-15 11:46:34 +0100 | [diff] [blame] | 285 | ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] |
| 286 | .endm |
| 287 | |
| 288 | .macro set_loaded_vcpu vcpu, ctxt, tmp |
Marc Zyngier | 14ef9d0 | 2020-09-30 14:05:35 +0100 | [diff] [blame] | 289 | adr_this_cpu \ctxt, kvm_hyp_ctxt, \tmp |
Andrew Scull | 7db2153 | 2020-09-15 11:46:34 +0100 | [diff] [blame] | 290 | str \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] |
| 291 | .endm |
| 292 | |
James Morse | e9ee186 | 2020-08-21 15:07:05 +0100 | [diff] [blame] | 293 | /* |
| 294 | * KVM extable for unexpected exceptions. |
Mark Rutland | ae2b2f3 | 2021-10-19 17:02:10 +0100 | [diff] [blame] | 295 | * Create a struct kvm_exception_table_entry output to a section that can be |
| 296 | * mapped by EL2. The table is not sorted. |
| 297 | * |
| 298 | * The caller must ensure: |
James Morse | e9ee186 | 2020-08-21 15:07:05 +0100 | [diff] [blame] | 299 | * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented |
| 300 | * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup. |
| 301 | */ |
| 302 | .macro _kvm_extable, from, to |
| 303 | .pushsection __kvm_ex_table, "a" |
| 304 | .align 3 |
| 305 | .long (\from - .), (\to - .) |
| 306 | .popsection |
| 307 | .endm |
| 308 | |
Andrew Scull | 603d2bd | 2020-09-15 11:46:35 +0100 | [diff] [blame] | 309 | #define CPU_XREG_OFFSET(x) (CPU_USER_PT_REGS + 8*x) |
| 310 | #define CPU_LR_OFFSET CPU_XREG_OFFSET(30) |
| 311 | #define CPU_SP_EL0_OFFSET (CPU_LR_OFFSET + 8) |
| 312 | |
| 313 | /* |
| 314 | * We treat x18 as callee-saved as the host may use it as a platform |
| 315 | * register (e.g. for shadow call stack). |
| 316 | */ |
| 317 | .macro save_callee_saved_regs ctxt |
| 318 | str x18, [\ctxt, #CPU_XREG_OFFSET(18)] |
| 319 | stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)] |
| 320 | stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)] |
| 321 | stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)] |
| 322 | stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)] |
| 323 | stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)] |
| 324 | stp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)] |
| 325 | .endm |
| 326 | |
| 327 | .macro restore_callee_saved_regs ctxt |
| 328 | // We require \ctxt is not x18-x28 |
| 329 | ldr x18, [\ctxt, #CPU_XREG_OFFSET(18)] |
| 330 | ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)] |
| 331 | ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)] |
| 332 | ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)] |
| 333 | ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)] |
| 334 | ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)] |
| 335 | ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)] |
| 336 | .endm |
| 337 | |
| 338 | .macro save_sp_el0 ctxt, tmp |
| 339 | mrs \tmp, sp_el0 |
| 340 | str \tmp, [\ctxt, #CPU_SP_EL0_OFFSET] |
| 341 | .endm |
| 342 | |
| 343 | .macro restore_sp_el0 ctxt, tmp |
| 344 | ldr \tmp, [\ctxt, #CPU_SP_EL0_OFFSET] |
| 345 | msr sp_el0, \tmp |
| 346 | .endm |
| 347 | |
Marc Zyngier | fd9fc9f | 2012-12-10 11:16:40 +0000 | [diff] [blame] | 348 | #endif |
| 349 | |
| 350 | #endif /* __ARM_KVM_ASM_H__ */ |