arm64: kasan: mte: move GCR_EL1 switch to task switch when KASAN disabled
It is not necessary to write to GCR_EL1 on every kernel entry and
exit when HW tag-based KASAN is disabled because the kernel will not
execute any IRG instructions in that mode. Since accessing GCR_EL1
can be expensive on some microarchitectures, avoid doing so by moving
the access to task switch when HW tag-based KASAN is disabled.
Signed-off-by: Peter Collingbourne <pcc@google.com>
Acked-by: Andrey Konovalov <andreyknvl@gmail.com>
Link: https://linux-review.googlesource.com/id/I78e90d60612a94c24344526f476ac4ff216e10d2
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20210924010655.2886918-1-pcc@google.com
Signed-off-by: Will Deacon <will@kernel.org>
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index bc6d5a9..2f69ae4 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -168,9 +168,9 @@
.macro mte_set_kernel_gcr, tmp, tmp2
#ifdef CONFIG_KASAN_HW_TAGS
-alternative_if_not ARM64_MTE
+alternative_cb kasan_hw_tags_enable
b 1f
-alternative_else_nop_endif
+alternative_cb_end
mov \tmp, KERNEL_GCR_EL1
msr_s SYS_GCR_EL1, \tmp
1:
@@ -178,10 +178,10 @@
.endm
.macro mte_set_user_gcr, tsk, tmp, tmp2
-#ifdef CONFIG_ARM64_MTE
-alternative_if_not ARM64_MTE
+#ifdef CONFIG_KASAN_HW_TAGS
+alternative_cb kasan_hw_tags_enable
b 1f
-alternative_else_nop_endif
+alternative_cb_end
ldr \tmp, [\tsk, #THREAD_MTE_CTRL]
mte_set_gcr \tmp, \tmp2
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index e5e801b..0cdae08 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -179,6 +179,30 @@ static void mte_update_sctlr_user(struct task_struct *task)
task->thread.sctlr_user = sctlr;
}
+static void mte_update_gcr_excl(struct task_struct *task)
+{
+ /*
+ * SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by
+ * mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled.
+ */
+ if (kasan_hw_tags_enabled())
+ return;
+
+ write_sysreg_s(
+ ((task->thread.mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
+ SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND,
+ SYS_GCR_EL1);
+}
+
+void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
+ __le32 *updptr, int nr_inst)
+{
+ BUG_ON(nr_inst != 1); /* Branch -> NOP */
+
+ if (kasan_hw_tags_enabled())
+ *updptr = cpu_to_le32(aarch64_insn_gen_nop());
+}
+
void mte_thread_init_user(void)
{
if (!system_supports_mte())
@@ -198,6 +222,7 @@ void mte_thread_switch(struct task_struct *next)
return;
mte_update_sctlr_user(next);
+ mte_update_gcr_excl(next);
/*
* Check if an async tag exception occurred at EL1.
@@ -243,6 +268,7 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg)
if (task == current) {
preempt_disable();
mte_update_sctlr_user(task);
+ mte_update_gcr_excl(task);
update_sctlr_el1(task->thread.sctlr_user);
preempt_enable();
}
