Neoverse N1 Errata Workaround 1542419
Coherent I-cache is causing a prefetch violation where when the core
executes an instruction that has recently been modified, the core might
fetch a stale instruction which violates the ordering of instruction
fetches.
The workaround includes an instruction sequence to implementation
defined registers to trap all EL0 IC IVAU instructions to EL3 and a trap
handler to execute a TLB inner-shareable invalidation to an arbitrary
address followed by a DSB.
Signed-off-by: Lauren Wehrmeister <lauren.wehrmeister@arm.com>
Change-Id: Ic3b7cbb11cf2eaf9005523ef5578a372593ae4d6
diff --git a/lib/cpus/aarch64/cpu_helpers.S b/lib/cpus/aarch64/cpu_helpers.S
index de1177c..808c7f8 100644
--- a/lib/cpus/aarch64/cpu_helpers.S
+++ b/lib/cpus/aarch64/cpu_helpers.S
@@ -227,6 +227,27 @@
ret
endfunc cpu_rev_var_hs
+/*
+ * Compare the CPU's revision-variant (x0) with a given range (x1 - x2), for errata
+ * application purposes. If the revision-variant is between or includes the given
+ * values, this indicates that errata applies; otherwise not.
+ *
+ * Shall clobber: x0-x4
+ */
+ .globl cpu_rev_var_range
+func cpu_rev_var_range
+ mov x3, #ERRATA_APPLIES
+ mov x4, #ERRATA_NOT_APPLIES
+ cmp x0, x1
+ csel x1, x3, x4, hs
+ cbz x1, 1f
+ cmp x0, x2
+ csel x1, x3, x4, ls
+1:
+ mov x0, x1
+ ret
+endfunc cpu_rev_var_range
+
#if REPORT_ERRATA
/*
* void print_errata_status(void);
diff --git a/lib/cpus/aarch64/neoverse_n1.S b/lib/cpus/aarch64/neoverse_n1.S
index 31e7a3a..c9bb005 100644
--- a/lib/cpus/aarch64/neoverse_n1.S
+++ b/lib/cpus/aarch64/neoverse_n1.S
@@ -9,6 +9,7 @@
#include <neoverse_n1.h>
#include <cpuamu.h>
#include <cpu_macros.S>
+#include <context.h>
/* Hardware handled coherency */
#if HW_ASSISTED_COHERENCY == 0
@@ -20,6 +21,10 @@
#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+#if ERRATA_N1_IC_TRAP
+ .global neoverse_n1_errata_ic_trap_handler
+#endif
+
/* --------------------------------------------------
* Errata Workaround for Neoverse N1 Erratum 1043202.
* This applies to revision r0p0 and r1p0 of Neoverse N1.
@@ -337,6 +342,41 @@
b cpu_rev_var_ls
endfunc check_errata_1315703
+/* --------------------------------------------------
+ * Errata Workaround for Neoverse N1 Erratum 1542419.
+ * This applies to revisions r3p0 - r4p0 of Neoverse N1
+ * Inputs:
+ * x0: variant[4:7] and revision[0:3] of current cpu.
+ * Shall clobber: x0-x17
+ * --------------------------------------------------
+ */
+func errata_n1_1542419_wa
+ /* Compare x0 against revision r3p0 and r4p0 */
+ mov x17, x30
+ bl check_errata_1542419
+ cbz x0, 1f
+
+ /* Apply instruction patching sequence */
+ ldr x0, =0x0
+ msr CPUPSELR_EL3, x0
+ ldr x0, =0xEE670D35
+ msr CPUPOR_EL3, x0
+ ldr x0, =0xFFFF0FFF
+ msr CPUPMR_EL3, x0
+ ldr x0, =0x08000020007D
+ msr CPUPCR_EL3, x0
+ isb
+1:
+ ret x17
+endfunc errata_n1_1542419_wa
+
+func check_errata_1542419
+ /* Applies to everything r3p0 - r4p0. */
+ mov x1, #0x30
+ mov x2, #0x40
+ b cpu_rev_var_range
+endfunc check_errata_1542419
+
func neoverse_n1_reset_func
mov x19, x30
@@ -406,6 +446,11 @@
bl errata_n1_1315703_wa
#endif
+#if ERRATA_N1_1542419
+ mov x0, x18
+ bl errata_n1_1542419_wa
+#endif
+
#if ENABLE_AMU
/* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */
mrs x0, actlr_el3
@@ -471,12 +516,49 @@
report_errata ERRATA_N1_1262888, neoverse_n1, 1262888
report_errata ERRATA_N1_1275112, neoverse_n1, 1275112
report_errata ERRATA_N1_1315703, neoverse_n1, 1315703
+ report_errata ERRATA_N1_1542419, neoverse_n1, 1542419
report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184
ldp x8, x30, [sp], #16
ret
endfunc neoverse_n1_errata_report
#endif
+
+/*
+ * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB
+ * inner-shareable invalidation to an arbitrary address followed by a DSB.
+ *
+ * x1: Exception Syndrome
+ */
+func neoverse_n1_errata_ic_trap_handler
+ cmp x1, #NEOVERSE_N1_EC_IC_TRAP
+ b.ne 1f
+ tlbi vae3is, xzr
+ dsb sy
+
+ # Skip the IC instruction itself
+ mrs x3, elr_el3
+ add x3, x3, #4
+ msr elr_el3, x3
+
+ ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
+ ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
+ ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
+ ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
+
+#if IMAGE_BL31 && RAS_EXTENSION
+ /*
+ * Issue Error Synchronization Barrier to synchronize SErrors before
+ * exiting EL3. We're running with EAs unmasked, so any synchronized
+ * errors would be taken immediately; therefore no need to inspect
+ * DISR_EL1 register.
+ */
+ esb
+#endif
+ eret
+1:
+ ret
+endfunc neoverse_n1_errata_ic_trap_handler
/* ---------------------------------------------
* This function provides neoverse_n1 specific
@@ -497,6 +579,7 @@
ret
endfunc neoverse_n1_cpu_reg_dump
-declare_cpu_ops neoverse_n1, NEOVERSE_N1_MIDR, \
+declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \
neoverse_n1_reset_func, \
+ neoverse_n1_errata_ic_trap_handler, \
neoverse_n1_core_pwr_dwn