[pkvm] Statically link el2.bin built at -O0
Spit out a statically linked el2.bin file under arch/arm64/kvm/hyp/nvhe/
which can be used for the purposes of analysis.
Signed-off-by: Will Deacon <will@kernel.org>
diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h
index bdf1f6b..dea9bc4 100644
--- a/arch/arm64/include/asm/alternative-macros.h
+++ b/arch/arm64/include/asm/alternative-macros.h
@@ -221,20 +221,580 @@
#ifndef __ASSEMBLY__
+#include <linux/bug.h>
#include <linux/types.h>
+#if ARM64_NCAPS != 88
+#error Fatal peramifier malfunction; more internships needed.
+#endif
+
+#define __PERAMIFY_LIKELY(N) \
+ asm_volatile_goto( \
+ ALTERNATIVE_CB("b %l[l_no]", N, alt_cb_patch_nops) \
+ :::: l_no)
+
+#define PERAMIFIER_LIKELY(N) \
+ switch (N) { \
+ case 0: \
+ __PERAMIFY_LIKELY(0); \
+ break; \
+ case 1: \
+ __PERAMIFY_LIKELY(1); \
+ break; \
+ case 2: \
+ __PERAMIFY_LIKELY(2); \
+ break; \
+ case 3: \
+ __PERAMIFY_LIKELY(3); \
+ break; \
+ case 4: \
+ __PERAMIFY_LIKELY(4); \
+ break; \
+ case 5: \
+ __PERAMIFY_LIKELY(5); \
+ break; \
+ case 6: \
+ __PERAMIFY_LIKELY(6); \
+ break; \
+ case 7: \
+ __PERAMIFY_LIKELY(7); \
+ break; \
+ case 8: \
+ __PERAMIFY_LIKELY(8); \
+ break; \
+ case 9: \
+ __PERAMIFY_LIKELY(9); \
+ break; \
+ case 10: \
+ __PERAMIFY_LIKELY(10); \
+ break; \
+ case 11: \
+ __PERAMIFY_LIKELY(11); \
+ break; \
+ case 12: \
+ __PERAMIFY_LIKELY(12); \
+ break; \
+ case 13: \
+ __PERAMIFY_LIKELY(13); \
+ break; \
+ case 14: \
+ __PERAMIFY_LIKELY(14); \
+ break; \
+ case 15: \
+ __PERAMIFY_LIKELY(15); \
+ break; \
+ case 16: \
+ __PERAMIFY_LIKELY(16); \
+ break; \
+ case 17: \
+ __PERAMIFY_LIKELY(17); \
+ break; \
+ case 18: \
+ __PERAMIFY_LIKELY(18); \
+ break; \
+ case 19: \
+ __PERAMIFY_LIKELY(19); \
+ break; \
+ case 20: \
+ __PERAMIFY_LIKELY(20); \
+ break; \
+ case 21: \
+ __PERAMIFY_LIKELY(21); \
+ break; \
+ case 22: \
+ __PERAMIFY_LIKELY(22); \
+ break; \
+ case 23: \
+ __PERAMIFY_LIKELY(23); \
+ break; \
+ case 24: \
+ __PERAMIFY_LIKELY(24); \
+ break; \
+ case 25: \
+ __PERAMIFY_LIKELY(25); \
+ break; \
+ case 26: \
+ __PERAMIFY_LIKELY(26); \
+ break; \
+ case 27: \
+ __PERAMIFY_LIKELY(27); \
+ break; \
+ case 28: \
+ __PERAMIFY_LIKELY(28); \
+ break; \
+ case 29: \
+ __PERAMIFY_LIKELY(29); \
+ break; \
+ case 30: \
+ __PERAMIFY_LIKELY(30); \
+ break; \
+ case 31: \
+ __PERAMIFY_LIKELY(31); \
+ break; \
+ case 32: \
+ __PERAMIFY_LIKELY(32); \
+ break; \
+ case 33: \
+ __PERAMIFY_LIKELY(33); \
+ break; \
+ case 34: \
+ __PERAMIFY_LIKELY(34); \
+ break; \
+ case 35: \
+ __PERAMIFY_LIKELY(35); \
+ break; \
+ case 36: \
+ __PERAMIFY_LIKELY(36); \
+ break; \
+ case 37: \
+ __PERAMIFY_LIKELY(37); \
+ break; \
+ case 38: \
+ __PERAMIFY_LIKELY(38); \
+ break; \
+ case 39: \
+ __PERAMIFY_LIKELY(39); \
+ break; \
+ case 40: \
+ __PERAMIFY_LIKELY(40); \
+ break; \
+ case 41: \
+ __PERAMIFY_LIKELY(41); \
+ break; \
+ case 42: \
+ __PERAMIFY_LIKELY(42); \
+ break; \
+ case 43: \
+ __PERAMIFY_LIKELY(43); \
+ break; \
+ case 44: \
+ __PERAMIFY_LIKELY(44); \
+ break; \
+ case 45: \
+ __PERAMIFY_LIKELY(45); \
+ break; \
+ case 46: \
+ __PERAMIFY_LIKELY(46); \
+ break; \
+ case 47: \
+ __PERAMIFY_LIKELY(47); \
+ break; \
+ case 48: \
+ __PERAMIFY_LIKELY(48); \
+ break; \
+ case 49: \
+ __PERAMIFY_LIKELY(49); \
+ break; \
+ case 50: \
+ __PERAMIFY_LIKELY(50); \
+ break; \
+ case 51: \
+ __PERAMIFY_LIKELY(51); \
+ break; \
+ case 52: \
+ __PERAMIFY_LIKELY(52); \
+ break; \
+ case 53: \
+ __PERAMIFY_LIKELY(53); \
+ break; \
+ case 54: \
+ __PERAMIFY_LIKELY(54); \
+ break; \
+ case 55: \
+ __PERAMIFY_LIKELY(55); \
+ break; \
+ case 56: \
+ __PERAMIFY_LIKELY(56); \
+ break; \
+ case 57: \
+ __PERAMIFY_LIKELY(57); \
+ break; \
+ case 58: \
+ __PERAMIFY_LIKELY(58); \
+ break; \
+ case 59: \
+ __PERAMIFY_LIKELY(59); \
+ break; \
+ case 60: \
+ __PERAMIFY_LIKELY(60); \
+ break; \
+ case 61: \
+ __PERAMIFY_LIKELY(61); \
+ break; \
+ case 62: \
+ __PERAMIFY_LIKELY(62); \
+ break; \
+ case 63: \
+ __PERAMIFY_LIKELY(63); \
+ break; \
+ case 64: \
+ __PERAMIFY_LIKELY(64); \
+ break; \
+ case 65: \
+ __PERAMIFY_LIKELY(65); \
+ break; \
+ case 66: \
+ __PERAMIFY_LIKELY(66); \
+ break; \
+ case 67: \
+ __PERAMIFY_LIKELY(67); \
+ break; \
+ case 68: \
+ __PERAMIFY_LIKELY(68); \
+ break; \
+ case 69: \
+ __PERAMIFY_LIKELY(69); \
+ break; \
+ case 70: \
+ __PERAMIFY_LIKELY(70); \
+ break; \
+ case 71: \
+ __PERAMIFY_LIKELY(71); \
+ break; \
+ case 72: \
+ __PERAMIFY_LIKELY(72); \
+ break; \
+ case 73: \
+ __PERAMIFY_LIKELY(73); \
+ break; \
+ case 74: \
+ __PERAMIFY_LIKELY(74); \
+ break; \
+ case 75: \
+ __PERAMIFY_LIKELY(75); \
+ break; \
+ case 76: \
+ __PERAMIFY_LIKELY(76); \
+ break; \
+ case 77: \
+ __PERAMIFY_LIKELY(77); \
+ break; \
+ case 78: \
+ __PERAMIFY_LIKELY(78); \
+ break; \
+ case 79: \
+ __PERAMIFY_LIKELY(79); \
+ break; \
+ case 80: \
+ __PERAMIFY_LIKELY(80); \
+ break; \
+ case 81: \
+ __PERAMIFY_LIKELY(81); \
+ break; \
+ case 82: \
+ __PERAMIFY_LIKELY(82); \
+ break; \
+ case 83: \
+ __PERAMIFY_LIKELY(83); \
+ break; \
+ case 84: \
+ __PERAMIFY_LIKELY(84); \
+ break; \
+ case 85: \
+ __PERAMIFY_LIKELY(85); \
+ break; \
+ case 86: \
+ __PERAMIFY_LIKELY(86); \
+ break; \
+ case 87: \
+ __PERAMIFY_LIKELY(87); \
+ break; \
+ default: \
+ BUG(); \
+ }
+
+#define __PERAMIFY_UNLIKELY(N) \
+ asm_volatile_goto( \
+ ALTERNATIVE("nop", "b %l[l_yes]", N) \
+ :::: l_yes)
+
+
+#define PERAMIFIER_UNLIKELY(N) \
+ switch (N) { \
+ case 0: \
+ __PERAMIFY_UNLIKELY(0); \
+ break; \
+ case 1: \
+ __PERAMIFY_UNLIKELY(1); \
+ break; \
+ case 2: \
+ __PERAMIFY_UNLIKELY(2); \
+ break; \
+ case 3: \
+ __PERAMIFY_UNLIKELY(3); \
+ break; \
+ case 4: \
+ __PERAMIFY_UNLIKELY(4); \
+ break; \
+ case 5: \
+ __PERAMIFY_UNLIKELY(5); \
+ break; \
+ case 6: \
+ __PERAMIFY_UNLIKELY(6); \
+ break; \
+ case 7: \
+ __PERAMIFY_UNLIKELY(7); \
+ break; \
+ case 8: \
+ __PERAMIFY_UNLIKELY(8); \
+ break; \
+ case 9: \
+ __PERAMIFY_UNLIKELY(9); \
+ break; \
+ case 10: \
+ __PERAMIFY_UNLIKELY(10); \
+ break; \
+ case 11: \
+ __PERAMIFY_UNLIKELY(11); \
+ break; \
+ case 12: \
+ __PERAMIFY_UNLIKELY(12); \
+ break; \
+ case 13: \
+ __PERAMIFY_UNLIKELY(13); \
+ break; \
+ case 14: \
+ __PERAMIFY_UNLIKELY(14); \
+ break; \
+ case 15: \
+ __PERAMIFY_UNLIKELY(15); \
+ break; \
+ case 16: \
+ __PERAMIFY_UNLIKELY(16); \
+ break; \
+ case 17: \
+ __PERAMIFY_UNLIKELY(17); \
+ break; \
+ case 18: \
+ __PERAMIFY_UNLIKELY(18); \
+ break; \
+ case 19: \
+ __PERAMIFY_UNLIKELY(19); \
+ break; \
+ case 20: \
+ __PERAMIFY_UNLIKELY(20); \
+ break; \
+ case 21: \
+ __PERAMIFY_UNLIKELY(21); \
+ break; \
+ case 22: \
+ __PERAMIFY_UNLIKELY(22); \
+ break; \
+ case 23: \
+ __PERAMIFY_UNLIKELY(23); \
+ break; \
+ case 24: \
+ __PERAMIFY_UNLIKELY(24); \
+ break; \
+ case 25: \
+ __PERAMIFY_UNLIKELY(25); \
+ break; \
+ case 26: \
+ __PERAMIFY_UNLIKELY(26); \
+ break; \
+ case 27: \
+ __PERAMIFY_UNLIKELY(27); \
+ break; \
+ case 28: \
+ __PERAMIFY_UNLIKELY(28); \
+ break; \
+ case 29: \
+ __PERAMIFY_UNLIKELY(29); \
+ break; \
+ case 30: \
+ __PERAMIFY_UNLIKELY(30); \
+ break; \
+ case 31: \
+ __PERAMIFY_UNLIKELY(31); \
+ break; \
+ case 32: \
+ __PERAMIFY_UNLIKELY(32); \
+ break; \
+ case 33: \
+ __PERAMIFY_UNLIKELY(33); \
+ break; \
+ case 34: \
+ __PERAMIFY_UNLIKELY(34); \
+ break; \
+ case 35: \
+ __PERAMIFY_UNLIKELY(35); \
+ break; \
+ case 36: \
+ __PERAMIFY_UNLIKELY(36); \
+ break; \
+ case 37: \
+ __PERAMIFY_UNLIKELY(37); \
+ break; \
+ case 38: \
+ __PERAMIFY_UNLIKELY(38); \
+ break; \
+ case 39: \
+ __PERAMIFY_UNLIKELY(39); \
+ break; \
+ case 40: \
+ __PERAMIFY_UNLIKELY(40); \
+ break; \
+ case 41: \
+ __PERAMIFY_UNLIKELY(41); \
+ break; \
+ case 42: \
+ __PERAMIFY_UNLIKELY(42); \
+ break; \
+ case 43: \
+ __PERAMIFY_UNLIKELY(43); \
+ break; \
+ case 44: \
+ __PERAMIFY_UNLIKELY(44); \
+ break; \
+ case 45: \
+ __PERAMIFY_UNLIKELY(45); \
+ break; \
+ case 46: \
+ __PERAMIFY_UNLIKELY(46); \
+ break; \
+ case 47: \
+ __PERAMIFY_UNLIKELY(47); \
+ break; \
+ case 48: \
+ __PERAMIFY_UNLIKELY(48); \
+ break; \
+ case 49: \
+ __PERAMIFY_UNLIKELY(49); \
+ break; \
+ case 50: \
+ __PERAMIFY_UNLIKELY(50); \
+ break; \
+ case 51: \
+ __PERAMIFY_UNLIKELY(51); \
+ break; \
+ case 52: \
+ __PERAMIFY_UNLIKELY(52); \
+ break; \
+ case 53: \
+ __PERAMIFY_UNLIKELY(53); \
+ break; \
+ case 54: \
+ __PERAMIFY_UNLIKELY(54); \
+ break; \
+ case 55: \
+ __PERAMIFY_UNLIKELY(55); \
+ break; \
+ case 56: \
+ __PERAMIFY_UNLIKELY(56); \
+ break; \
+ case 57: \
+ __PERAMIFY_UNLIKELY(57); \
+ break; \
+ case 58: \
+ __PERAMIFY_UNLIKELY(58); \
+ break; \
+ case 59: \
+ __PERAMIFY_UNLIKELY(59); \
+ break; \
+ case 60: \
+ __PERAMIFY_UNLIKELY(60); \
+ break; \
+ case 61: \
+ __PERAMIFY_UNLIKELY(61); \
+ break; \
+ case 62: \
+ __PERAMIFY_UNLIKELY(62); \
+ break; \
+ case 63: \
+ __PERAMIFY_UNLIKELY(63); \
+ break; \
+ case 64: \
+ __PERAMIFY_UNLIKELY(64); \
+ break; \
+ case 65: \
+ __PERAMIFY_UNLIKELY(65); \
+ break; \
+ case 66: \
+ __PERAMIFY_UNLIKELY(66); \
+ break; \
+ case 67: \
+ __PERAMIFY_UNLIKELY(67); \
+ break; \
+ case 68: \
+ __PERAMIFY_UNLIKELY(68); \
+ break; \
+ case 69: \
+ __PERAMIFY_UNLIKELY(69); \
+ break; \
+ case 70: \
+ __PERAMIFY_UNLIKELY(70); \
+ break; \
+ case 71: \
+ __PERAMIFY_UNLIKELY(71); \
+ break; \
+ case 72: \
+ __PERAMIFY_UNLIKELY(72); \
+ break; \
+ case 73: \
+ __PERAMIFY_UNLIKELY(73); \
+ break; \
+ case 74: \
+ __PERAMIFY_UNLIKELY(74); \
+ break; \
+ case 75: \
+ __PERAMIFY_UNLIKELY(75); \
+ break; \
+ case 76: \
+ __PERAMIFY_UNLIKELY(76); \
+ break; \
+ case 77: \
+ __PERAMIFY_UNLIKELY(77); \
+ break; \
+ case 78: \
+ __PERAMIFY_UNLIKELY(78); \
+ break; \
+ case 79: \
+ __PERAMIFY_UNLIKELY(79); \
+ break; \
+ case 80: \
+ __PERAMIFY_UNLIKELY(80); \
+ break; \
+ case 81: \
+ __PERAMIFY_UNLIKELY(81); \
+ break; \
+ case 82: \
+ __PERAMIFY_UNLIKELY(82); \
+ break; \
+ case 83: \
+ __PERAMIFY_UNLIKELY(83); \
+ break; \
+ case 84: \
+ __PERAMIFY_UNLIKELY(84); \
+ break; \
+ case 85: \
+ __PERAMIFY_UNLIKELY(85); \
+ break; \
+ case 86: \
+ __PERAMIFY_UNLIKELY(86); \
+ break; \
+ case 87: \
+ __PERAMIFY_UNLIKELY(87); \
+ break; \
+ default: \
+ BUG(); \
+ }
+
static __always_inline bool
alternative_has_feature_likely(const unsigned long feature)
{
compiletime_assert(feature < ARM64_NCAPS,
"feature must be < ARM64_NCAPS");
+#ifdef __KVM_NVHE_HYPERVISOR__
+ PERAMIFIER_LIKELY(feature);
+#else
asm_volatile_goto(
ALTERNATIVE_CB("b %l[l_no]", %[feature], alt_cb_patch_nops)
:
: [feature] "i" (feature)
:
: l_no);
+#endif
return true;
l_no:
@@ -247,12 +807,16 @@
compiletime_assert(feature < ARM64_NCAPS,
"feature must be < ARM64_NCAPS");
+#ifdef __KVM_NVHE_HYPERVISOR__
+ PERAMIFIER_UNLIKELY(feature);
+#else
asm_volatile_goto(
ALTERNATIVE("nop", "b %l[l_yes]", %[feature])
:
: [feature] "i" (feature)
:
: l_yes);
+#endif
return false;
l_yes:
diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
index 48ddc0f..e40ba35 100644
--- a/arch/arm64/include/asm/jump_label.h
+++ b/arch/arm64/include/asm/jump_label.h
@@ -15,6 +15,12 @@
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
+#ifdef __KVM_NVHE_HYPERVISOR__
+#define __static_key_enabled(k) (!!__READ_ONCE((k)->enabled.counter))
+#define arch_static_branch(k,b) (b ^ __static_key_enabled(k))
+#define arch_static_branch_jump(k,b) (b ^ __static_key_enabled(k))
+#else
+
static __always_inline bool arch_static_branch(struct static_key * const key,
const bool branch)
{
@@ -49,5 +55,6 @@
return true;
}
+#endif /* !__KVM_NVHE_HYPERVISOR__ */
#endif /* __ASSEMBLY__ */
#endif /* __ASM_JUMP_LABEL_H */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 65af544..9cd9ffa 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -590,7 +590,7 @@
#define __build_check_flag(v, flagset, f, m) \
do { \
- typeof(v->arch.flagset) *_fset; \
+ typeof(v->arch.flagset) * __maybe_unused _fset; \
\
/* Check that the flags fit in the mask */ \
BUILD_BUG_ON(HWEIGHT(m) != HWEIGHT((f) | (m))); \
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 53a7290..3dbeea0 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -68,10 +68,8 @@
KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
-#ifdef CONFIG_ARM64_PSEUDO_NMI
/* Static key checked in GIC_PRIO_IRQOFF. */
KVM_NVHE_ALIAS(gic_nonsecure_priorities);
-#endif
/* EL2 exception handling */
KVM_NVHE_ALIAS(__start___kvm_ex_table);
@@ -116,6 +114,9 @@
KVM_NVHE_ALIAS(kvm_hyp_debug_uart_set_basep);
#endif
+/* Only needed to link when built at -O0 */
+KVM_NVHE_ALIAS(cpu_hwcaps);
+
#endif /* CONFIG_KVM */
#ifdef CONFIG_EFI_ZBOOT
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 530347c..fe57e03 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -110,3 +110,10 @@
# Skip objtool checking for this directory because nVHE code is compiled with
# non-standard build rules.
OBJECT_FILES_NON_STANDARD := y
+
+# Hacks for analysis purposes
+ccflags-y += -O0
+extra-y += dummy.o el2.bin
+LDFLAGS_el2.bin := -T
+$(obj)/el2.bin: $(obj)/hyp.lds $(addprefix $(obj)/,$(hyp-obj)) $(obj)/dummy.o FORCE
+ $(call if_changed,ld)
diff --git a/arch/arm64/kvm/hyp/nvhe/dummy.c b/arch/arm64/kvm/hyp/nvhe/dummy.c
new file mode 100644
index 0000000..6eb28bc
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/dummy.c
@@ -0,0 +1,88 @@
+// Dummy definitions to produce a fully linked EL2 binary for analysis.
+
+#define KVM_NVHE_ALIAS(sym) unsigned long sym
+#define KVM_NVHE_ALIAS_HYP(x,y) unsigned long __kvm_nvhe_##y
+
+// XXX: Following imported directly from image-vars.h
+///////////////////////////////////////////////////////
+
+/*
+ * KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_, to
+ * separate it from the kernel proper. The following symbols are legally
+ * accessed by it, therefore provide aliases to make them linkable.
+ * Do not include symbols which may not be safely accessed under hypervisor
+ * memory mappings.
+ */
+
+/* Alternative callbacks for init-time patching of nVHE hyp code. */
+KVM_NVHE_ALIAS(kvm_patch_vector_branch);
+KVM_NVHE_ALIAS(kvm_update_va_mask);
+KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
+KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);
+KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
+KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
+KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
+KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
+KVM_NVHE_ALIAS(alt_cb_patch_nops);
+
+/* Global kernel state accessed by nVHE hyp code. */
+KVM_NVHE_ALIAS(kvm_vgic_global_state);
+
+/* Kernel symbols used to call panic() from nVHE hyp code (via ERET). */
+KVM_NVHE_ALIAS(nvhe_hyp_panic_handler);
+
+/* Vectors installed by hyp-init on reset HVC. */
+KVM_NVHE_ALIAS(__hyp_stub_vectors);
+
+/* Static keys which are set if a vGIC trap should be handled in hyp. */
+KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
+KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
+
+/* Static key checked in GIC_PRIO_IRQOFF. */
+KVM_NVHE_ALIAS(gic_nonsecure_priorities);
+
+/* EL2 exception handling */
+KVM_NVHE_ALIAS(__start___kvm_ex_table);
+KVM_NVHE_ALIAS(__stop___kvm_ex_table);
+
+/* PMU available static key */
+#ifdef CONFIG_HW_PERF_EVENTS
+KVM_NVHE_ALIAS(kvm_arm_pmu_available);
+#endif
+
+/* Position-independent library routines */
+KVM_NVHE_ALIAS_HYP(clear_page, __pi_clear_page);
+KVM_NVHE_ALIAS_HYP(copy_page, __pi_copy_page);
+KVM_NVHE_ALIAS_HYP(memcpy, __pi_memcpy);
+KVM_NVHE_ALIAS_HYP(memset, __pi_memset);
+
+#ifdef CONFIG_KASAN
+KVM_NVHE_ALIAS_HYP(__memcpy, __pi_memcpy);
+KVM_NVHE_ALIAS_HYP(__memset, __pi_memset);
+#endif
+
+/* Kernel memory sections */
+KVM_NVHE_ALIAS(__start_rodata);
+KVM_NVHE_ALIAS(__end_rodata);
+KVM_NVHE_ALIAS(__bss_start);
+KVM_NVHE_ALIAS(__bss_stop);
+
+/* Hyp memory sections */
+KVM_NVHE_ALIAS(__hyp_idmap_text_start);
+KVM_NVHE_ALIAS(__hyp_idmap_text_end);
+KVM_NVHE_ALIAS(__hyp_text_start);
+KVM_NVHE_ALIAS(__hyp_text_end);
+KVM_NVHE_ALIAS(__hyp_bss_start);
+KVM_NVHE_ALIAS(__hyp_bss_end);
+KVM_NVHE_ALIAS(__hyp_rodata_start);
+KVM_NVHE_ALIAS(__hyp_rodata_end);
+
+/* pKVM static key */
+KVM_NVHE_ALIAS(kvm_protected_mode_initialized);
+
+#ifdef CONFIG_KVM_ARM_HYP_DEBUG_UART
+KVM_NVHE_ALIAS(kvm_hyp_debug_uart_set_basep);
+#endif
+
+/* Only needed to link when built at -O0 */
+KVM_NVHE_ALIAS(cpu_hwcaps);
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 2eafde2..a62c519 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -29,6 +29,21 @@
*/
static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu);
+bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int bits)
+{
+ unsigned int k;
+ unsigned int lim = bits/BITS_PER_LONG;
+ unsigned long result = 0;
+
+ for (k = 0; k < lim; k++)
+ result |= (dst[k] = bitmap1[k] & bitmap2[k]);
+ if (bits % BITS_PER_LONG)
+ result |= (dst[k] = bitmap1[k] & bitmap2[k] &
+ BITMAP_LAST_WORD_MASK(bits));
+ return result != 0;
+}
+
/*
* Set trap register values based on features in ID_AA64PFR0.
*/
