Implement workaround for AT speculative behaviour
During context switching from higher EL (EL2 or higher)
to lower EL can cause incorrect translation in TLB due to
speculative execution of AT instruction using out-of-context
translation regime.
Workaround is implemented as below during EL's (EL1 or EL2)
"context_restore" operation:
1. Disable page table walk using SCTLR.M and TCR.EPD0 & EPD1
bits for EL1 or EL2 (stage1 and stage2 disabled)
2. Save all system registers except TCR and SCTLR (for EL1 and EL2)
3. Do memory barrier operation (isb) to ensure all
system register writes are done.
4. Restore TCR and SCTLR registers (for EL1 and EL2)
Errata details are available for various CPUs as below:
Cortex-A76: 1165522
Cortex-A72: 1319367
Cortex-A57: 1319537
Cortex-A55: 1530923
Cortex-A53: 1530924
More details can be found in mail-chain:
https://lists.trustedfirmware.org/pipermail/tf-a/2020-April/000445.html
Currently, Workaround is implemented as build option which is default
disabled.
Signed-off-by: Manish V Badarkhe <Manish.Badarkhe@arm.com>
Change-Id: If8545e61f782cb0c2dda7ffbaf50681c825bd2f0
diff --git a/Makefile b/Makefile
index 5c4f36c..9972362 100644
--- a/Makefile
+++ b/Makefile
@@ -891,6 +891,7 @@
$(eval $(call assert_boolean,USE_SPINLOCK_CAS))
$(eval $(call assert_boolean,ENCRYPT_BL31))
$(eval $(call assert_boolean,ENCRYPT_BL32))
+$(eval $(call assert_boolean,ERRATA_SPECULATIVE_AT))
$(eval $(call assert_numeric,ARM_ARCH_MAJOR))
$(eval $(call assert_numeric,ARM_ARCH_MINOR))
@@ -967,6 +968,7 @@
$(eval $(call add_define,BL2_IN_XIP_MEM))
$(eval $(call add_define,BL2_INV_DCACHE))
$(eval $(call add_define,USE_SPINLOCK_CAS))
+$(eval $(call add_define,ERRATA_SPECULATIVE_AT))
ifeq (${SANITIZE_UB},trap)
$(eval $(call add_define,MONITOR_TRAPS))
diff --git a/docs/getting_started/build-options.rst b/docs/getting_started/build-options.rst
index 90fe83f..6f3b605 100644
--- a/docs/getting_started/build-options.rst
+++ b/docs/getting_started/build-options.rst
@@ -673,6 +673,29 @@
default value of this flag is ``no``. Note this option must be enabled only
for ARM architecture greater than Armv8.5-A.
+- ``ERRATA_SPECULATIVE_AT``: This flag enables/disables page table walk during
+ context restore as speculative AT instructions using an out-of-context
+ translation regime could cause subsequent requests to generate an incorrect
+ translation.
+ System registers are not updated during context save, hence this workaround
+ need not be applied in the context save path.
+
+ This boolean option enables errata for all below CPUs.
+
+ +---------+--------------+
+ | Errata | CPU |
+ +=========+==============+
+ | 1165522 | Cortex-A76 |
+ +---------+--------------+
+ | 1319367 | Cortex-A72 |
+ +---------+--------------+
+ | 1319537 | Cortex-A57 |
+ +---------+--------------+
+ | 1530923 | Cortex-A55 |
+ +---------+--------------+
+ | 1530924 | Cortex-A53 |
+ +---------+--------------+
+
GICv3 driver options
--------------------
diff --git a/include/arch/aarch64/arch.h b/include/arch/aarch64/arch.h
index e45a594..81e0f27 100644
--- a/include/arch/aarch64/arch.h
+++ b/include/arch/aarch64/arch.h
@@ -381,6 +381,7 @@
/* HCR definitions */
#define HCR_API_BIT (ULL(1) << 41)
#define HCR_APK_BIT (ULL(1) << 40)
+#define HCR_E2H_BIT (ULL(1) << 34)
#define HCR_TGE_BIT (ULL(1) << 27)
#define HCR_RW_SHIFT U(31)
#define HCR_RW_BIT (ULL(1) << HCR_RW_SHIFT)
diff --git a/lib/el3_runtime/aarch64/context.S b/lib/el3_runtime/aarch64/context.S
index 221f33e..984468a 100644
--- a/lib/el3_runtime/aarch64/context.S
+++ b/lib/el3_runtime/aarch64/context.S
@@ -234,6 +234,21 @@
*/
func el2_sysregs_context_restore
+#if ERRATA_SPECULATIVE_AT
+/* Clear EPD0 and EPD1 bit and M bit to disable PTW */
+ mrs x9, hcr_el2
+ tst x9, #HCR_E2H_BIT
+ bne 1f
+ mrs x9, tcr_el2
+ orr x9, x9, #TCR_EPD0_BIT
+ orr x9, x9, #TCR_EPD1_BIT
+ msr tcr_el2, x9
+1: mrs x9, sctlr_el2
+ bic x9, x9, #SCTLR_M_BIT
+ msr sctlr_el2, x9
+ isb
+#endif
+
ldp x9, x10, [x0, #CTX_ACTLR_EL2]
msr actlr_el2, x9
msr afsr0_el2, x10
@@ -282,17 +297,15 @@
msr mair_el2, x15
msr mdcr_el2, x16
- ldp x17, x9, [x0, #CTX_PMSCR_EL2]
+ ldr x17, [x0, #CTX_PMSCR_EL2]
msr PMSCR_EL2, x17
- msr sctlr_el2, x9
ldp x10, x11, [x0, #CTX_SPSR_EL2]
msr spsr_el2, x10
msr sp_el2, x11
- ldp x12, x13, [x0, #CTX_TCR_EL2]
- msr tcr_el2, x12
- msr tpidr_el2, x13
+ ldr x12, [x0, #CTX_TPIDR_EL2]
+ msr tpidr_el2, x12
ldp x14, x15, [x0, #CTX_TTBR0_EL2]
msr ttbr0_el2, x14
@@ -404,6 +417,19 @@
msr scxtnum_el2, x9
#endif
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Make sure all registers are stored successfully except
+ * SCTLR_EL2 and TCR_EL2
+ */
+ isb
+#endif
+
+ ldr x9, [x0, #CTX_SCTLR_EL2]
+ msr sctlr_el2, x9
+ ldr x9, [x0, #CTX_TCR_EL2]
+ msr tcr_el2, x9
+
ret
endfunc el2_sysregs_context_restore
@@ -515,12 +541,22 @@
*/
func el1_sysregs_context_restore
+#if ERRATA_SPECULATIVE_AT
+ mrs x9, tcr_el1
+ orr x9, x9, #TCR_EPD0_BIT
+ orr x9, x9, #TCR_EPD1_BIT
+ msr tcr_el1, x9
+ mrs x9, sctlr_el1
+ bic x9, x9, #SCTLR_M_BIT
+ msr sctlr_el1, x9
+ isb
+#endif
+
ldp x9, x10, [x0, #CTX_SPSR_EL1]
msr spsr_el1, x9
msr elr_el1, x10
- ldp x15, x16, [x0, #CTX_SCTLR_EL1]
- msr sctlr_el1, x15
+ ldr x16, [x0, #CTX_ACTLR_EL1]
msr actlr_el1, x16
ldp x17, x9, [x0, #CTX_CPACR_EL1]
@@ -539,9 +575,8 @@
msr mair_el1, x14
msr amair_el1, x15
- ldp x16, x17, [x0, #CTX_TCR_EL1]
- msr tcr_el1, x16
- msr tpidr_el1, x17
+ ldr x16,[x0, #CTX_TPIDR_EL1]
+ msr tpidr_el1, x16
ldp x9, x10, [x0, #CTX_TPIDR_EL0]
msr tpidr_el0, x9
@@ -597,6 +632,19 @@
msr GCR_EL1, x14
#endif
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Make sure all registers are stored successfully except
+ * SCTLR_EL1 and TCR_EL1
+ */
+ isb
+#endif
+
+ ldr x9, [x0, #CTX_SCTLR_EL1]
+ msr sctlr_el1, x9
+ ldr x9, [x0, #CTX_TCR_EL1]
+ msr tcr_el1, x9
+
/* No explict ISB required here as ERET covers it */
ret
endfunc el1_sysregs_context_restore
diff --git a/make_helpers/defaults.mk b/make_helpers/defaults.mk
index 590a800..608e963 100644
--- a/make_helpers/defaults.mk
+++ b/make_helpers/defaults.mk
@@ -293,3 +293,6 @@
# than Armv8.5-A
# By default it is set to "no"
SUPPORT_STACK_MEMTAG := no
+
+# Select workaround for AT speculative behaviour.
+ERRATA_SPECULATIVE_AT := 0