arch/arm64/kvm/hyp/hyp-entry.S - linux - Git at Google

 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (C) 2015-2018 - ARM Ltd
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  */

 #include <linux/arm-smccc.h>
 #include <linux/linkage.h>

 #include <asm/alternative.h>
 #include <asm/assembler.h>
 #include <asm/cpufeature.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmu.h>
 #include <asm/mmu.h>

 	.text

 el1_sync:
 	mrs	x0, esr_el2
 	lsr	x0, x0, #ESR_ELx_EC_SHIFT
 	cmp	x0, #ESR_ELx_EC_HVC64
 	ccmp	x0, #ESR_ELx_EC_HVC32, #4, ne
 	b.ne	el1_trap

 	/*
 	 * Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
 	 * The workaround has already been applied on the host,
 	 * so let's quickly get back to the guest. We don't bother
 	 * restoring x1, as it can be clobbered anyway.
 	 */
 	ldr	x1, [sp]				// vcpu's x0
 	eor	w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
 	cbz	w1, wa_epilogue

 	/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
 	eor	w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
 			  ARM_SMCCC_ARCH_WORKAROUND_2)
 	cbnz	w1, el1_trap

 #ifdef CONFIG_ARM64_SSBD
 alternative_cb	arm64_enable_wa2_handling
 	b	wa2_end
 alternative_cb_end
 	ldr_this_cpu x2, kvm_hyp_running_vcpu, x0
 	ldr	x0, [x2, #VCPU_WORKAROUND_FLAGS]

 	// Sanitize the argument and update the guest flags
 	ldr	x1, [sp, #8]			// vcpu's x1
 	clz	w1, w1				// Murphy's device:
 	lsr	w1, w1, #5			// w1 = !!w1 without using
 	eor	w1, w1, #1			// the flags...
 	bfi	x0, x1, #VCPU_WORKAROUND_2_FLAG_SHIFT, #1
 	str	x0, [x2, #VCPU_WORKAROUND_FLAGS]

 	/* Check that we actually need to perform the call */
 	ldr_this_cpu x0, arm64_ssbd_callback_required, x2
 	cbz	x0, wa2_end

 	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_2
 	smc	#0

 	/* Don't leak data from the SMC call */
 	mov	x3, xzr
 wa2_end:
 	mov	x2, xzr
 	mov	x1, xzr
 #endif

 wa_epilogue:
 	mov	x0, xzr
 	add	sp, sp, #16
 	eret
 	sb

 el1_trap:
 	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
 	mov	x0, #ARM_EXCEPTION_TRAP
 	b	__guest_exit

 el1_irq:
 	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
 	mov	x0, #ARM_EXCEPTION_IRQ
 	b	__guest_exit

 el1_error:
 	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
 	mov	x0, #ARM_EXCEPTION_EL1_SERROR
 	b	__guest_exit

 el2_sync:
 	/* Check for illegal exception return, otherwise panic */
 	mrs	x0, spsr_el2

 	/* if this was something else, then panic! */
 	tst	x0, #PSR_IL_BIT
 	b.eq	hyp_panic

 	/* Let's attempt a recovery from the illegal exception return */
 	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
 	mov	x0, #ARM_EXCEPTION_IL
 	b	__guest_exit


 el2_error:
 	ldp	x0, x1, [sp], #16

 	/*
 	 * Only two possibilities:
 	 * 1) Either we come from the exit path, having just unmasked
 	 *    PSTATE.A: change the return code to an EL2 fault, and
 	 *    carry on, as we're already in a sane state to handle it.
 	 * 2) Or we come from anywhere else, and that's a bug: we panic.
 	 *
 	 * For (1), x0 contains the original return code and x1 doesn't
 	 * contain anything meaningful at that stage. We can reuse them
 	 * as temp registers.
 	 * For (2), who cares?
 	 */
 	mrs	x0, elr_el2
 	adr	x1, abort_guest_exit_start
 	cmp	x0, x1
 	adr	x1, abort_guest_exit_end
 	ccmp	x0, x1, #4, ne
 	b.ne	hyp_panic
 	eret
 	sb

 #ifdef __KVM_NVHE_HYPERVISOR__
 SYM_FUNC_START(__hyp_do_panic)
 	mov	lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
 		      PSR_MODE_EL1h)
 	msr	spsr_el2, lr
 	ldr	lr, =panic
 	msr	elr_el2, lr
 	eret
 	sb
 SYM_FUNC_END(__hyp_do_panic)
 #endif

 .macro invalid_vector	label, target = hyp_panic
 	.align	2
 SYM_CODE_START(\label)
 	b \target
 SYM_CODE_END(\label)
 .endm

 	/* None of these should ever happen */
 	invalid_vector	el2t_sync_invalid
 	invalid_vector	el2t_irq_invalid
 	invalid_vector	el2t_fiq_invalid
 	invalid_vector	el2t_error_invalid
 	invalid_vector	el2h_sync_invalid
 	invalid_vector	el2h_irq_invalid
 	invalid_vector	el2h_fiq_invalid
 	invalid_vector	el1_fiq_invalid

 	.ltorg

 	.align 11

 .macro check_preamble_length start, end
 /* kvm_patch_vector_branch() generates code that jumps over the preamble. */
 .if ((\end-\start) != KVM_VECTOR_PREAMBLE)
 	.error "KVM vector preamble length mismatch"
 .endif
 .endm

 .macro valid_vect target
 	.align 7
 661:
 	esb
 	stp	x0, x1, [sp, #-16]!
 662:
 	b	\target

 check_preamble_length 661b, 662b
 .endm

 .macro invalid_vect target
 	.align 7
 661:
 	b	\target
 	nop
 662:
 	ldp	x0, x1, [sp], #16
 	b	\target

 check_preamble_length 661b, 662b
 .endm

 SYM_CODE_START(__kvm_hyp_vector)
 	invalid_vect	el2t_sync_invalid	// Synchronous EL2t
 	invalid_vect	el2t_irq_invalid	// IRQ EL2t
 	invalid_vect	el2t_fiq_invalid	// FIQ EL2t
 	invalid_vect	el2t_error_invalid	// Error EL2t

 	valid_vect	el2_sync		// Synchronous EL2h
 	invalid_vect	el2h_irq_invalid	// IRQ EL2h
 	invalid_vect	el2h_fiq_invalid	// FIQ EL2h
 	valid_vect	el2_error		// Error EL2h

 	valid_vect	el1_sync		// Synchronous 64-bit EL1
 	valid_vect	el1_irq			// IRQ 64-bit EL1
 	invalid_vect	el1_fiq_invalid		// FIQ 64-bit EL1
 	valid_vect	el1_error		// Error 64-bit EL1

 	valid_vect	el1_sync		// Synchronous 32-bit EL1
 	valid_vect	el1_irq			// IRQ 32-bit EL1
 	invalid_vect	el1_fiq_invalid		// FIQ 32-bit EL1
 	valid_vect	el1_error		// Error 32-bit EL1
 SYM_CODE_END(__kvm_hyp_vector)

 #ifdef CONFIG_KVM_INDIRECT_VECTORS
 .macro hyp_ventry
 	.align 7
 1:	esb
 	.rept 26
 	nop
 	.endr
 /*
  * The default sequence is to directly branch to the KVM vectors,
  * using the computed offset. This applies for VHE as well as
  * !ARM64_HARDEN_EL2_VECTORS. The first vector must always run the preamble.
  *
  * For ARM64_HARDEN_EL2_VECTORS configurations, this gets replaced
  * with:
  *
  * stp	x0, x1, [sp, #-16]!
  * movz	x0, #(addr & 0xffff)
  * movk	x0, #((addr >> 16) & 0xffff), lsl #16
  * movk	x0, #((addr >> 32) & 0xffff), lsl #32
  * br	x0
  *
  * Where:
  * addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + KVM_VECTOR_PREAMBLE.
  * See kvm_patch_vector_branch for details.
  */
 alternative_cb	kvm_patch_vector_branch
 	stp	x0, x1, [sp, #-16]!
 	b	__kvm_hyp_vector + (1b - 0b + KVM_VECTOR_PREAMBLE)
 	nop
 	nop
 	nop
 alternative_cb_end
 .endm

 .macro generate_vectors
 0:
 	.rept 16
 	hyp_ventry
 	.endr
 	.org 0b + SZ_2K		// Safety measure
 .endm

 	.align	11
 SYM_CODE_START(__bp_harden_hyp_vecs)
 	.rept BP_HARDEN_EL2_SLOTS
 	generate_vectors
 	.endr
 1:	.org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ
 	.org 1b
 SYM_CODE_END(__bp_harden_hyp_vecs)
 #endif
	/* SPDX-License-Identifier: GPL-2.0-only */
	/*
	* Copyright (C) 2015-2018 - ARM Ltd
	* Author: Marc Zyngier <marc.zyngier@arm.com>
	*/

	#include <linux/arm-smccc.h>
	#include <linux/linkage.h>

	#include <asm/alternative.h>
	#include <asm/assembler.h>
	#include <asm/cpufeature.h>
	#include <asm/kvm_arm.h>
	#include <asm/kvm_asm.h>
	#include <asm/kvm_mmu.h>
	#include <asm/mmu.h>

	.text

	el1_sync:
	mrs x0, esr_el2
	lsr x0, x0, #ESR_ELx_EC_SHIFT
	cmp x0, #ESR_ELx_EC_HVC64
	ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
	b.ne el1_trap

	/*
	* Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
	* The workaround has already been applied on the host,
	* so let's quickly get back to the guest. We don't bother
	* restoring x1, as it can be clobbered anyway.
	*/
	ldr x1, [sp] // vcpu's x0
	eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
	cbz w1, wa_epilogue

	/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
	eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
	ARM_SMCCC_ARCH_WORKAROUND_2)
	cbnz w1, el1_trap

	#ifdef CONFIG_ARM64_SSBD
	alternative_cb arm64_enable_wa2_handling
	b wa2_end
	alternative_cb_end
	ldr_this_cpu x2, kvm_hyp_running_vcpu, x0
	ldr x0, [x2, #VCPU_WORKAROUND_FLAGS]

	// Sanitize the argument and update the guest flags
	ldr x1, [sp, #8] // vcpu's x1
	clz w1, w1 // Murphy's device:
	lsr w1, w1, #5 // w1 = !!w1 without using
	eor w1, w1, #1 // the flags...
	bfi x0, x1, #VCPU_WORKAROUND_2_FLAG_SHIFT, #1
	str x0, [x2, #VCPU_WORKAROUND_FLAGS]

	/* Check that we actually need to perform the call */
	ldr_this_cpu x0, arm64_ssbd_callback_required, x2
	cbz x0, wa2_end

	mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
	smc #0

	/* Don't leak data from the SMC call */
	mov x3, xzr
	wa2_end:
	mov x2, xzr
	mov x1, xzr
	#endif

	wa_epilogue:
	mov x0, xzr
	add sp, sp, #16
	eret
	sb

	el1_trap:
	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
	mov x0, #ARM_EXCEPTION_TRAP
	b __guest_exit

	el1_irq:
	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
	mov x0, #ARM_EXCEPTION_IRQ
	b __guest_exit

	el1_error:
	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
	mov x0, #ARM_EXCEPTION_EL1_SERROR
	b __guest_exit

	el2_sync:
	/* Check for illegal exception return, otherwise panic */
	mrs x0, spsr_el2

	/* if this was something else, then panic! */
	tst x0, #PSR_IL_BIT
	b.eq hyp_panic

	/* Let's attempt a recovery from the illegal exception return */
	ldr_this_cpu x1, kvm_hyp_running_vcpu, x0
	mov x0, #ARM_EXCEPTION_IL
	b __guest_exit


	el2_error:
	ldp x0, x1, [sp], #16

	/*
	* Only two possibilities:
	* 1) Either we come from the exit path, having just unmasked
	* PSTATE.A: change the return code to an EL2 fault, and
	* carry on, as we're already in a sane state to handle it.
	* 2) Or we come from anywhere else, and that's a bug: we panic.
	*
	* For (1), x0 contains the original return code and x1 doesn't
	* contain anything meaningful at that stage. We can reuse them
	* as temp registers.
	* For (2), who cares?
	*/
	mrs x0, elr_el2
	adr x1, abort_guest_exit_start
	cmp x0, x1
	adr x1, abort_guest_exit_end
	ccmp x0, x1, #4, ne
	b.ne hyp_panic
	eret
	sb

	#ifdef __KVM_NVHE_HYPERVISOR__
	SYM_FUNC_START(__hyp_do_panic)
	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
	PSR_MODE_EL1h)
	msr spsr_el2, lr
	ldr lr, =panic
	msr elr_el2, lr
	eret
	sb
	SYM_FUNC_END(__hyp_do_panic)
	#endif

	.macro invalid_vector label, target = hyp_panic
	.align 2
	SYM_CODE_START(\label)
	b \target
	SYM_CODE_END(\label)
	.endm

	/* None of these should ever happen */
	invalid_vector el2t_sync_invalid
	invalid_vector el2t_irq_invalid
	invalid_vector el2t_fiq_invalid
	invalid_vector el2t_error_invalid
	invalid_vector el2h_sync_invalid
	invalid_vector el2h_irq_invalid
	invalid_vector el2h_fiq_invalid
	invalid_vector el1_fiq_invalid

	.ltorg

	.align 11

	.macro check_preamble_length start, end
	/* kvm_patch_vector_branch() generates code that jumps over the preamble. */
	.if ((\end-\start) != KVM_VECTOR_PREAMBLE)
	.error "KVM vector preamble length mismatch"
	.endif
	.endm

	.macro valid_vect target
	.align 7
	661:
	esb
	stp x0, x1, [sp, #-16]!
	662:
	b \target

	check_preamble_length 661b, 662b
	.endm

	.macro invalid_vect target
	.align 7
	661:
	b \target
	nop
	662:
	ldp x0, x1, [sp], #16
	b \target

	check_preamble_length 661b, 662b
	.endm

	SYM_CODE_START(__kvm_hyp_vector)
	invalid_vect el2t_sync_invalid // Synchronous EL2t
	invalid_vect el2t_irq_invalid // IRQ EL2t
	invalid_vect el2t_fiq_invalid // FIQ EL2t
	invalid_vect el2t_error_invalid // Error EL2t

	valid_vect el2_sync // Synchronous EL2h
	invalid_vect el2h_irq_invalid // IRQ EL2h
	invalid_vect el2h_fiq_invalid // FIQ EL2h
	valid_vect el2_error // Error EL2h

	valid_vect el1_sync // Synchronous 64-bit EL1
	valid_vect el1_irq // IRQ 64-bit EL1
	invalid_vect el1_fiq_invalid // FIQ 64-bit EL1
	valid_vect el1_error // Error 64-bit EL1

	valid_vect el1_sync // Synchronous 32-bit EL1
	valid_vect el1_irq // IRQ 32-bit EL1
	invalid_vect el1_fiq_invalid // FIQ 32-bit EL1
	valid_vect el1_error // Error 32-bit EL1
	SYM_CODE_END(__kvm_hyp_vector)

	#ifdef CONFIG_KVM_INDIRECT_VECTORS
	.macro hyp_ventry
	.align 7
	1: esb
	.rept 26
	nop
	.endr
	/*
	* The default sequence is to directly branch to the KVM vectors,
	* using the computed offset. This applies for VHE as well as
	* !ARM64_HARDEN_EL2_VECTORS. The first vector must always run the preamble.
	*
	* For ARM64_HARDEN_EL2_VECTORS configurations, this gets replaced
	* with:
	*
	* stp x0, x1, [sp, #-16]!
	* movz x0, #(addr & 0xffff)
	* movk x0, #((addr >> 16) & 0xffff), lsl #16
	* movk x0, #((addr >> 32) & 0xffff), lsl #32
	* br x0
	*
	* Where:
	* addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + KVM_VECTOR_PREAMBLE.
	* See kvm_patch_vector_branch for details.
	*/
	alternative_cb kvm_patch_vector_branch
	stp x0, x1, [sp, #-16]!
	b __kvm_hyp_vector + (1b - 0b + KVM_VECTOR_PREAMBLE)
	nop
	nop
	nop
	alternative_cb_end
	.endm

	.macro generate_vectors
	0:
	.rept 16
	hyp_ventry
	.endr
	.org 0b + SZ_2K // Safety measure
	.endm

	.align 11
	SYM_CODE_START(__bp_harden_hyp_vecs)
	.rept BP_HARDEN_EL2_SLOTS
	generate_vectors
	.endr
	1: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ
	.org 1b
	SYM_CODE_END(__bp_harden_hyp_vecs)
	#endif