feat(sve): enable SVE for the secure world
Enables SVE support for the secure world via ENABLE_SVE_FOR_SWD.
ENABLE_SVE_FOR_SWD defaults to 0 and has to be explicitly set by the
platform. SVE is configured during initial setup and then uses EL3
context save/restore routine to switch between SVE configurations for
different contexts.
Reset value of CPTR_EL3 changed to be most restrictive by default.
Signed-off-by: Max Shvetsov <maksims.svecovs@arm.com>
Change-Id: I889fbbc2e435435d66779b73a2d90d1188bf4116
diff --git a/lib/el3_runtime/aarch64/context.S b/lib/el3_runtime/aarch64/context.S
index 0ec9ffd..d610fd4 100644
--- a/lib/el3_runtime/aarch64/context.S
+++ b/lib/el3_runtime/aarch64/context.S
@@ -901,6 +901,29 @@
msr spsr_el3, x16
msr elr_el3, x17
+#if IMAGE_BL31
+ /* ----------------------------------------------------------
+ * Restore CPTR_EL3, ZCR_EL3 for SVE support.
+ * If SVE is not supported - skip the restoration.
+ * ZCR is only restored if SVE is supported and enabled.
+ * Synchronization is required before zcr_el3 is addressed.
+ * ----------------------------------------------------------
+ */
+ mrs x17, id_aa64pfr0_el1
+ ubfx x17, x17, ID_AA64PFR0_SVE_SHIFT, ID_AA64PFR0_SVE_LENGTH
+ cbz x17, sve_not_enabled
+
+ ldp x19, x20, [sp, #CTX_EL3STATE_OFFSET + CTX_CPTR_EL3]
+ msr cptr_el3, x19
+
+ ands x19, x19, #CPTR_EZ_BIT
+ beq sve_not_enabled
+
+ isb
+ msr S3_6_C1_C2_0, x20 /* zcr_el3 */
+sve_not_enabled:
+#endif
+
#if IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639
/* ----------------------------------------------------------
* Restore mitigation state as it was on entry to EL3
diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c
index 96023b6..7a25151 100644
--- a/lib/el3_runtime/aarch64/context_mgmt.c
+++ b/lib/el3_runtime/aarch64/context_mgmt.c
@@ -178,8 +178,20 @@
* indicated by the interrupt routing model for BL31.
*/
scr_el3 |= get_scr_el3_from_routing_model(security_state);
+
+#if ENABLE_SVE_FOR_NS
+ if (security_state == NON_SECURE) {
+ sve_enable(ctx);
+ }
+#endif
+#if ENABLE_SVE_FOR_SWD
+ if (security_state == SECURE) {
+ sve_enable(ctx);
+ }
#endif
+#endif
+
/*
* SCR_EL3.HCE: Enable HVC instructions if next execution state is
* AArch64 and next EL is EL2, or if next execution state is AArch32 and
@@ -334,10 +346,6 @@
amu_enable(el2_unused);
#endif
-#if ENABLE_SVE_FOR_NS
- sve_enable(el2_unused);
-#endif
-
#if ENABLE_MPAM_FOR_LOWER_ELS
mpam_enable(el2_unused);
#endif
diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c
index fa4ac77..7043cc2 100644
--- a/lib/extensions/sve/sve.c
+++ b/lib/extensions/sve/sve.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -11,7 +11,13 @@
#include <lib/el3_runtime/pubsub.h>
#include <lib/extensions/sve.h>
-bool sve_supported(void)
+/*
+ * Converts SVE vector size restriction in bytes to LEN according to ZCR_EL3 documentation.
+ * VECTOR_SIZE = (LEN+1) * 128
+ */
+#define CONVERT_SVE_LENGTH(x) (((x / 128) - 1))
+
+static bool sve_supported(void)
{
uint64_t features;
@@ -19,113 +25,19 @@
return (features & ID_AA64PFR0_SVE_MASK) == 1U;
}
-static void *disable_sve_hook(const void *arg)
-{
- uint64_t cptr;
-
- if (!sve_supported())
- return (void *)-1;
-
- /*
- * Disable SVE, SIMD and FP access for the Secure world.
- * As the SIMD/FP registers are part of the SVE Z-registers, any
- * use of SIMD/FP functionality will corrupt the SVE registers.
- * Therefore it is necessary to prevent use of SIMD/FP support
- * in the Secure world as well as SVE functionality.
- */
- cptr = read_cptr_el3();
- cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT);
- write_cptr_el3(cptr);
-
- /*
- * No explicit ISB required here as ERET to switch to Secure
- * world covers it
- */
- return (void *)0;
-}
-
-static void *enable_sve_hook(const void *arg)
-{
- uint64_t cptr;
-
- if (!sve_supported())
- return (void *)-1;
-
- /*
- * Enable SVE, SIMD and FP access for the Non-secure world.
- */
- cptr = read_cptr_el3();
- cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT);
- write_cptr_el3(cptr);
-
- /*
- * No explicit ISB required here as ERET to switch to Non-secure
- * world covers it
- */
- return (void *)0;
-}
-
-void sve_enable(bool el2_unused)
+void sve_enable(cpu_context_t *context)
{
- uint64_t cptr;
-
- if (!sve_supported())
+ if (!sve_supported()) {
return;
-
-#if CTX_INCLUDE_FPREGS
- /*
- * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems.
- */
- assert(0);
-#endif
- /*
- * Update CPTR_EL3 to enable access to SVE functionality for the
- * Non-secure world.
- * NOTE - assumed that CPTR_EL3.TFP is set to allow access to
- * the SIMD, floating-point and SVE support.
- *
- * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality
- * in the Non-secure world.
- */
- cptr = read_cptr_el3();
- cptr |= CPTR_EZ_BIT;
- write_cptr_el3(cptr);
-
- /*
- * Need explicit ISB here to guarantee that update to ZCR_ELx
- * and CPTR_EL2.TZ do not result in trap to EL3.
- */
- isb();
+ }
- /*
- * Ensure lower ELs have access to full vector length.
- */
- write_zcr_el3(ZCR_EL3_LEN_MASK);
+ u_register_t cptr_el3 = read_cptr_el3();
- if (el2_unused) {
- /*
- * Update CPTR_EL2 to enable access to SVE functionality
- * for Non-secure world, EL2 and Non-secure EL1 and EL0.
- * NOTE - assumed that CPTR_EL2.TFP is set to allow
- * access to the SIMD, floating-point and SVE support.
- *
- * CPTR_EL2.TZ: Set to 0 to enable access to SVE support
- * for EL2 and Non-secure EL1 and EL0.
- */
- cptr = read_cptr_el2();
- cptr &= ~(CPTR_EL2_TZ_BIT);
- write_cptr_el2(cptr);
+ /* Enable access to SVE functionality for all ELs. */
+ cptr_el3 = (cptr_el3 | CPTR_EZ_BIT) & ~(TFP_BIT);
+ write_ctx_reg(get_el3state_ctx(context), CTX_CPTR_EL3, cptr_el3);
- /*
- * Ensure lower ELs have access to full vector length.
- */
- write_zcr_el2(ZCR_EL2_LEN_MASK);
- }
- /*
- * No explicit ISB required here as ERET to switch to
- * Non-secure world covers it.
- */
+ /* Restrict maximum SVE vector length (SVE_VECTOR_LENGTH+1) * 128. */
+ write_ctx_reg(get_el3state_ctx(context), CTX_ZCR_EL3,
+ (ZCR_EL3_LEN_MASK & CONVERT_SVE_LENGTH(512)));
}
-
-SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook);
-SUBSCRIBE_TO_EVENT(cm_entering_normal_world, enable_sve_hook);