perf(cpus): inline cpu_rev_var checks
We strive to apply errata as close to reset as possible with as few
things enabled as possible. Importantly, the I-cache will not be
enabled. This means that repeated branches to these tiny functions must
be re-fetched all the way from memory each time which has glacial speed.
Cores are allowed to fetch things ahead of time though as long as
execution is fairly linear. So we can trade a little bit of space (3 to
7 instructions per erratum) to keep things linear and not have to go to
memory.
While we're at it, optimise the the cpu_rev_var_{ls, hs, range}
functions to take up less space. Dropping the moves allows for a bit of
assembly magic that produces the same result in 2 and 3 instructions
respectively.
Change-Id: I51608352f23b2244ea7a99e76c10892d257f12bf
Signed-off-by: Boyan Karatotev <boyan.karatotev@arm.com>
diff --git a/include/lib/cpus/aarch64/cpu_macros.S b/include/lib/cpus/aarch64/cpu_macros.S
index ac26fd7..5a8219e 100644
--- a/include/lib/cpus/aarch64/cpu_macros.S
+++ b/include/lib/cpus/aarch64/cpu_macros.S
@@ -479,8 +479,35 @@
.endm
/*
- * Helpers to select which revisions errata apply to. Don't leave a link
- * register as the cpu_rev_var_*** will call the ret and we can save on one.
+ * Helpers to report if an erratum applies. Compares the given revision variant
+ * to the given value. Return ERRATA_APPLIES or ERRATA_NOT_APPLIES accordingly.
+ *
+ * _rev_num: the given revision variant. Or
+ * _rev_num_lo,_rev_num_hi: the lower and upper bounds of the revision variant
+ *
+ * in body:
+ * clobber: x0
+ * argument: x0 - cpu_rev_var
+ */
+.macro cpu_rev_var_ls _rev_num:req
+ cmp x0, #\_rev_num
+ cset x0, ls
+.endm
+
+.macro cpu_rev_var_hs _rev_num:req
+ cmp x0, #\_rev_num
+ cset x0, hs
+.endm
+
+.macro cpu_rev_var_range _rev_num_lo:req, _rev_num_hi:req
+ cmp x0, #\_rev_num_lo
+ mov x1, #\_rev_num_hi
+ ccmp x0, x1, #2, hs
+ cset x0, ls
+.endm
+
+/*
+ * Helpers to select which revisions errata apply to.
*
* _cpu:
* Name of cpu as given to declare_cpu_ops
@@ -496,28 +523,27 @@
* Revision to apply to
*
* in body:
- * clobber: x0 to x4
+ * clobber: x0 to x1
* argument: x0 - cpu_rev_var
*/
.macro check_erratum_ls _cpu:req, _cve:req, _id:req, _rev_num:req
func check_erratum_\_cpu\()_\_id
- mov x1, #\_rev_num
- b cpu_rev_var_ls
+ cpu_rev_var_ls \_rev_num
+ ret
endfunc check_erratum_\_cpu\()_\_id
.endm
.macro check_erratum_hs _cpu:req, _cve:req, _id:req, _rev_num:req
func check_erratum_\_cpu\()_\_id
- mov x1, #\_rev_num
- b cpu_rev_var_hs
+ cpu_rev_var_hs \_rev_num
+ ret
endfunc check_erratum_\_cpu\()_\_id
.endm
.macro check_erratum_range _cpu:req, _cve:req, _id:req, _rev_num_lo:req, _rev_num_hi:req
func check_erratum_\_cpu\()_\_id
- mov x1, #\_rev_num_lo
- mov x2, #\_rev_num_hi
- b cpu_rev_var_range
+ cpu_rev_var_range \_rev_num_lo, \_rev_num_hi
+ ret
endfunc check_erratum_\_cpu\()_\_id
.endm
@@ -532,7 +558,10 @@
endfunc check_erratum_\_cpu\()_\_id
.endm
-/* provide a shorthand for the name format for annoying errata */
+/*
+ * provide a shorthand for the name format for annoying errata
+ * body: clobber x0 to x3
+ */
.macro check_erratum_custom_start _cpu:req, _cve:req, _id:req
func check_erratum_\_cpu\()_\_id
.endm
diff --git a/lib/cpus/aarch64/cortex_a55.S b/lib/cpus/aarch64/cortex_a55.S
index f9e3eda..2267d66 100644
--- a/lib/cpus/aarch64/cortex_a55.S
+++ b/lib/cpus/aarch64/cortex_a55.S
@@ -51,8 +51,7 @@
check_erratum_custom_start cortex_a55, ERRATUM(778703)
mov x16, x30
- mov x1, #0x00
- bl cpu_rev_var_ls
+ cpu_rev_var_ls CPU_REV(0, 0)
/*
* Check that no private L2 cache is configured
*/
diff --git a/lib/cpus/aarch64/cortex_a76.S b/lib/cpus/aarch64/cortex_a76.S
index 1fd0789..589edce 100644
--- a/lib/cpus/aarch64/cortex_a76.S
+++ b/lib/cpus/aarch64/cortex_a76.S
@@ -345,11 +345,10 @@
check_erratum_custom_start cortex_a76, ERRATUM(1286807)
#if ERRATA_A76_1286807
mov x0, #ERRATA_APPLIES
- ret
#else
- mov x1, #0x30
- b cpu_rev_var_ls
+ cpu_rev_var_ls CPU_REV(3, 0)
#endif
+ ret
check_erratum_custom_end cortex_a76, ERRATUM(1286807)
workaround_reset_start cortex_a76, ERRATUM(1791580), ERRATA_A76_1791580
@@ -420,11 +419,10 @@
check_erratum_custom_start cortex_a76, ERRATUM(1165522)
#if ERRATA_A76_1165522
mov x0, #ERRATA_APPLIES
- ret
#else
- mov x1, #0x30
- b cpu_rev_var_ls
+ cpu_rev_var_ls CPU_REV(3, 0)
#endif
+ ret
check_erratum_custom_end cortex_a76, ERRATUM(1165522)
check_erratum_chosen cortex_a76, CVE(2022, 23960), WORKAROUND_CVE_2022_23960
diff --git a/lib/cpus/aarch64/cortex_a77.S b/lib/cpus/aarch64/cortex_a77.S
index 766bdc0..1759b77 100644
--- a/lib/cpus/aarch64/cortex_a77.S
+++ b/lib/cpus/aarch64/cortex_a77.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2018-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -36,8 +36,7 @@
workaround_reset_start cortex_a77, ERRATUM(1508412), ERRATA_A77_1508412
/* move cpu revision in again and compare against r0p0 */
mov x0, x7
- mov x1, #CPU_REV(0, 0)
- bl cpu_rev_var_ls
+ cpu_rev_var_ls CPU_REV(0, 0)
cbz x0, 1f
ldr x0, =0x0
diff --git a/lib/cpus/aarch64/cpu_helpers.S b/lib/cpus/aarch64/cpu_helpers.S
index 0f9a3b8..537f715 100644
--- a/lib/cpus/aarch64/cpu_helpers.S
+++ b/lib/cpus/aarch64/cpu_helpers.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2014-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -237,59 +237,6 @@
endfunc cpu_get_rev_var
/*
- * Compare the CPU's revision-variant (x0) with a given value (x1), for errata
- * application purposes. If the revision-variant is less than or same as a given
- * value, indicates that errata applies; otherwise not.
- *
- * Shall clobber: x0-x3
- */
- .globl cpu_rev_var_ls
-func cpu_rev_var_ls
- mov x2, #ERRATA_APPLIES
- mov x3, #ERRATA_NOT_APPLIES
- cmp x0, x1
- csel x0, x2, x3, ls
- ret
-endfunc cpu_rev_var_ls
-
-/*
- * Compare the CPU's revision-variant (x0) with a given value (x1), for errata
- * application purposes. If the revision-variant is higher than or same as a
- * given value, indicates that errata applies; otherwise not.
- *
- * Shall clobber: x0-x3
- */
- .globl cpu_rev_var_hs
-func cpu_rev_var_hs
- mov x2, #ERRATA_APPLIES
- mov x3, #ERRATA_NOT_APPLIES
- cmp x0, x1
- csel x0, x2, x3, hs
- ret
-endfunc cpu_rev_var_hs
-
-/*
- * Compare the CPU's revision-variant (x0) with a given range (x1 - x2), for errata
- * application purposes. If the revision-variant is between or includes the given
- * values, this indicates that errata applies; otherwise not.
- *
- * Shall clobber: x0-x4
- */
- .globl cpu_rev_var_range
-func cpu_rev_var_range
- mov x3, #ERRATA_APPLIES
- mov x4, #ERRATA_NOT_APPLIES
- cmp x0, x1
- csel x1, x3, x4, hs
- cbz x1, 1f
- cmp x0, x2
- csel x1, x3, x4, ls
-1:
- mov x0, x1
- ret
-endfunc cpu_rev_var_range
-
-/*
* int check_wa_cve_2017_5715(void);
*
* This function returns: