Merge pull request #1178 from davidcunado-arm/dc/enable_sve
Enable SVE for Non-secure world
diff --git a/Makefile b/Makefile
index fe75798..c0fddcf 100644
--- a/Makefile
+++ b/Makefile
@@ -468,6 +468,7 @@
$(eval $(call assert_boolean,ENABLE_PSCI_STAT))
$(eval $(call assert_boolean,ENABLE_RUNTIME_INSTRUMENTATION))
$(eval $(call assert_boolean,ENABLE_SPE_FOR_LOWER_ELS))
+$(eval $(call assert_boolean,ENABLE_SVE_FOR_NS))
$(eval $(call assert_boolean,ERROR_DEPRECATED))
$(eval $(call assert_boolean,GENERATE_COT))
$(eval $(call assert_boolean,GICV2_G0_FOR_EL3))
@@ -508,6 +509,7 @@
$(eval $(call add_define,ENABLE_PSCI_STAT))
$(eval $(call add_define,ENABLE_RUNTIME_INSTRUMENTATION))
$(eval $(call add_define,ENABLE_SPE_FOR_LOWER_ELS))
+$(eval $(call add_define,ENABLE_SVE_FOR_NS))
$(eval $(call add_define,ERROR_DEPRECATED))
$(eval $(call add_define,GICV2_G0_FOR_EL3))
$(eval $(call add_define,HW_ASSISTED_COHERENCY))
diff --git a/bl31/bl31.mk b/bl31/bl31.mk
index ebd0e71..fdcc931 100644
--- a/bl31/bl31.mk
+++ b/bl31/bl31.mk
@@ -54,6 +54,10 @@
BL31_SOURCES += lib/extensions/amu/aarch64/amu.c
endif
+ifeq (${ENABLE_SVE_FOR_NS},1)
+BL31_SOURCES += lib/extensions/sve/sve.c
+endif
+
BL31_LINKERFILE := bl31/bl31.ld.S
# Flag used to indicate if Crash reporting via console should be included
diff --git a/docs/user-guide.rst b/docs/user-guide.rst
index 138ed8b..13f0964 100644
--- a/docs/user-guide.rst
+++ b/docs/user-guide.rst
@@ -354,6 +354,17 @@
The default is 1 but is automatically disabled when the target architecture
is AArch32.
+- ``ENABLE_SVE_FOR_NS``: Boolean option to enable Scalable Vector Extension
+ (SVE) for the Non-secure world only. SVE is an optional architectural feature
+ for AArch64. Note that when SVE is enabled for the Non-secure world, access
+ to SIMD and floating-point functionality from the Secure world is disabled.
+ This is to avoid corruption of the Non-secure world data in the Z-registers
+ which are aliased by the SIMD and FP registers. The build option is not
+ compatible with the ``CTX_INCLUDE_FPREGS`` build option, and will raise an
+ assert on platforms where SVE is implemented and ``ENABLE_SVE_FOR_NS`` set to
+ 1. The default is 1 but is automatically disabled when the target
+ architecture is AArch32.
+
- ``ENABLE_STACK_PROTECTOR``: String option to enable the stack protection
checks in GCC. Allowed values are "all", "strong" and "0" (default).
"strong" is the recommended stack protection level if this feature is
diff --git a/include/common/aarch64/el3_common_macros.S b/include/common/aarch64/el3_common_macros.S
index ed35df8..63a0fa7 100644
--- a/include/common/aarch64/el3_common_macros.S
+++ b/include/common/aarch64/el3_common_macros.S
@@ -127,9 +127,9 @@
* CPTR_EL3.TTA: Set to zero so that System register accesses to the
* trace registers do not trap to EL3.
*
- * CPTR_EL3.TFP: Set to zero so that accesses to Advanced SIMD and
- * floating-point functionality do not trap to EL3.
- * ---------------------------------------------------------------------
+ * CPTR_EL3.TFP: Set to zero so that accesses to the V- or Z- registers
+ * by Advanced SIMD, floating-point or SVE instructions (if implemented)
+ * do not trap to EL3.
*/
mov_imm x0, (CPTR_EL3_RESET_VAL & ~(TCPAC_BIT | TTA_BIT | TFP_BIT))
msr cptr_el3, x0
diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h
index 65e9fc1..96e2d5f 100644
--- a/include/lib/aarch64/arch.h
+++ b/include/lib/aarch64/arch.h
@@ -114,6 +114,9 @@
#define ID_AA64PFR0_AMU_LENGTH U(4)
#define ID_AA64PFR0_AMU_MASK U(0xf)
#define ID_AA64PFR0_ELX_MASK U(0xf)
+#define ID_AA64PFR0_SVE_SHIFT U(32)
+#define ID_AA64PFR0_SVE_MASK U(0xf)
+#define ID_AA64PFR0_SVE_LENGTH U(4)
/* ID_AA64DFR0_EL1.PMS definitions (for ARMv8.2+) */
#define ID_AA64DFR0_PMS_SHIFT U(32)
@@ -301,6 +304,7 @@
#define TAM_BIT (U(1) << 30)
#define TTA_BIT (U(1) << 20)
#define TFP_BIT (U(1) << 10)
+#define CPTR_EZ_BIT (U(1) << 8)
#define CPTR_EL3_RESET_VAL U(0x0)
/* CPTR_EL2 definitions */
@@ -309,6 +313,7 @@
#define CPTR_EL2_TAM_BIT (U(1) << 30)
#define CPTR_EL2_TTA_BIT (U(1) << 20)
#define CPTR_EL2_TFP_BIT (U(1) << 10)
+#define CPTR_EL2_TZ_BIT (U(1) << 8)
#define CPTR_EL2_RESET_VAL CPTR_EL2_RES1
/* CPSR/SPSR definitions */
@@ -556,6 +561,18 @@
#define PMCR_EL0_D_BIT (U(1) << 3)
/*******************************************************************************
+ * Definitions for system register interface to SVE
+ ******************************************************************************/
+#define ZCR_EL3 S3_6_C1_C2_0
+#define ZCR_EL2 S3_4_C1_C2_0
+
+/* ZCR_EL3 definitions */
+#define ZCR_EL3_LEN_MASK U(0xf)
+
+/* ZCR_EL2 definitions */
+#define ZCR_EL2_LEN_MASK U(0xf)
+
+/*******************************************************************************
* Definitions of MAIR encodings for device and normal memory
******************************************************************************/
/*
diff --git a/include/lib/aarch64/arch_helpers.h b/include/lib/aarch64/arch_helpers.h
index b6be167..831dfb0 100644
--- a/include/lib/aarch64/arch_helpers.h
+++ b/include/lib/aarch64/arch_helpers.h
@@ -329,6 +329,9 @@
DEFINE_RENAME_SYSREG_RW_FUNCS(pmblimitr_el1, PMBLIMITR_EL1)
+DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el3, ZCR_EL3)
+DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el2, ZCR_EL2)
+
#define IS_IN_EL(x) \
(GET_EL(read_CurrentEl()) == MODE_EL##x)
diff --git a/include/lib/extensions/sve.h b/include/lib/extensions/sve.h
new file mode 100644
index 0000000..28923e3
--- /dev/null
+++ b/include/lib/extensions/sve.h
@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __SVE_H__
+#define __SVE_H__
+
+void sve_enable(int el2_unused);
+
+#endif /* __SVE_H__ */
diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c
index b892729..c6c2249 100644
--- a/lib/el3_runtime/aarch64/context_mgmt.c
+++ b/lib/el3_runtime/aarch64/context_mgmt.c
@@ -18,6 +18,7 @@
#include <smcc_helpers.h>
#include <spe.h>
#include <string.h>
+#include <sve.h>
#include <utils.h>
@@ -225,6 +226,10 @@
#if ENABLE_AMU
amu_enable(el2_unused);
#endif
+
+#if ENABLE_SVE_FOR_NS
+ sve_enable(el2_unused);
+#endif
#endif
}
diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c
new file mode 100644
index 0000000..14e51bd
--- /dev/null
+++ b/lib/extensions/sve/sve.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <arch.h>
+#include <arch_helpers.h>
+#include <pubsub.h>
+#include <sve.h>
+
+static void *disable_sve_hook(const void *arg)
+{
+ uint64_t features;
+
+ features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT;
+ if ((features & ID_AA64PFR0_SVE_MASK) == 1) {
+ uint64_t cptr;
+
+ /*
+ * Disable SVE, SIMD and FP access for the Secure world.
+ * As the SIMD/FP registers are part of the SVE Z-registers, any
+ * use of SIMD/FP functionality will corrupt the SVE registers.
+ * Therefore it is necessary to prevent use of SIMD/FP support
+ * in the Secure world as well as SVE functionality.
+ */
+ cptr = read_cptr_el3();
+ cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT);
+ write_cptr_el3(cptr);
+
+ /*
+ * No explicit ISB required here as ERET to switch to Secure
+ * world covers it
+ */
+ }
+ return 0;
+}
+
+static void *enable_sve_hook(const void *arg)
+{
+ uint64_t features;
+
+ features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT;
+ if ((features & ID_AA64PFR0_SVE_MASK) == 1) {
+ uint64_t cptr;
+
+ /*
+ * Enable SVE, SIMD and FP access for the Non-secure world.
+ */
+ cptr = read_cptr_el3();
+ cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT);
+ write_cptr_el3(cptr);
+
+ /*
+ * No explicit ISB required here as ERET to switch to Non-secure
+ * world covers it
+ */
+ }
+ return 0;
+}
+
+void sve_enable(int el2_unused)
+{
+ uint64_t features;
+
+ features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT;
+ if ((features & ID_AA64PFR0_SVE_MASK) == 1) {
+ uint64_t cptr;
+#if CTX_INCLUDE_FPREGS
+ /*
+ * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems.
+ */
+ assert(0);
+#endif
+ /*
+ * Update CPTR_EL3 to enable access to SVE functionality for the
+ * Non-secure world.
+ * NOTE - assumed that CPTR_EL3.TFP is set to allow access to
+ * the SIMD, floating-point and SVE support.
+ *
+ * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality
+ * in the Non-secure world.
+ */
+ cptr = read_cptr_el3();
+ cptr |= CPTR_EZ_BIT;
+ write_cptr_el3(cptr);
+
+ /*
+ * Need explicit ISB here to guarantee that update to ZCR_ELx
+ * and CPTR_EL2.TZ do not result in trap to EL3.
+ */
+ isb();
+
+ /*
+ * Ensure lower ELs have access to full vector length.
+ */
+ write_zcr_el3(ZCR_EL3_LEN_MASK);
+
+ if (el2_unused) {
+ /*
+ * Update CPTR_EL2 to enable access to SVE functionality
+ * for Non-secure world, EL2 and Non-secure EL1 and EL0.
+ * NOTE - assumed that CPTR_EL2.TFP is set to allow
+ * access to the SIMD, floating-point and SVE support.
+ *
+ * CPTR_EL2.TZ: Set to 0 to enable access to SVE support
+ * for EL2 and Non-secure EL1 and EL0.
+ */
+ cptr = read_cptr_el2();
+ cptr &= ~(CPTR_EL2_TZ_BIT);
+ write_cptr_el2(cptr);
+
+ /*
+ * Ensure lower ELs have access to full vector length.
+ */
+ write_zcr_el2(ZCR_EL2_LEN_MASK);
+ }
+ /*
+ * No explicit ISB required here as ERET to switch to
+ * Non-secure world covers it.
+ */
+ }
+}
+
+SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook);
+SUBSCRIBE_TO_EVENT(cm_entering_normal_world, enable_sve_hook);
diff --git a/make_helpers/defaults.mk b/make_helpers/defaults.mk
index 4584e63..fa0d17d 100644
--- a/make_helpers/defaults.mk
+++ b/make_helpers/defaults.mk
@@ -155,3 +155,12 @@
endif
ENABLE_AMU := 0
+
+# By default, enable Scalable Vector Extension if implemented for Non-secure
+# lower ELs
+# Note SVE is only supported on AArch64 - therefore do not enable in AArch32
+ifneq (${ARCH},aarch32)
+ ENABLE_SVE_FOR_NS := 1
+else
+ override ENABLE_SVE_FOR_NS := 0
+endif
diff --git a/plat/arm/board/juno/platform.mk b/plat/arm/board/juno/platform.mk
index bfb7847..fee4391 100644
--- a/plat/arm/board/juno/platform.mk
+++ b/plat/arm/board/juno/platform.mk
@@ -99,6 +99,9 @@
# Enable memory map related constants optimisation
ARM_BOARD_OPTIMISE_MEM := 1
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
+
include plat/arm/board/common/board_css.mk
include plat/arm/common/arm_common.mk
include plat/arm/soc/common/soc_css.mk
diff --git a/plat/compat/plat_compat.mk b/plat/compat/plat_compat.mk
index af88542..f1867da 100644
--- a/plat/compat/plat_compat.mk
+++ b/plat/compat/plat_compat.mk
@@ -18,3 +18,6 @@
BL31_SOURCES += plat/common/plat_psci_common.c \
plat/compat/plat_pm_compat.c \
plat/compat/plat_topology_compat.c
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/hisilicon/hikey/platform.mk b/plat/hisilicon/hikey/platform.mk
index 26218a4..18b5e15 100644
--- a/plat/hisilicon/hikey/platform.mk
+++ b/plat/hisilicon/hikey/platform.mk
@@ -24,6 +24,7 @@
PLAT_PL061_MAX_GPIOS := 160
COLD_BOOT_SINGLE_CPU := 1
PROGRAMMABLE_RESET_ADDRESS := 1
+ENABLE_SVE_FOR_NS := 0
# Process flags
$(eval $(call add_define,HIKEY_TSP_RAM_LOCATION_ID))
diff --git a/plat/hisilicon/hikey960/platform.mk b/plat/hisilicon/hikey960/platform.mk
index da7bb82..695f092 100644
--- a/plat/hisilicon/hikey960/platform.mk
+++ b/plat/hisilicon/hikey960/platform.mk
@@ -20,6 +20,7 @@
CRASH_CONSOLE_BASE := PL011_UART6_BASE
COLD_BOOT_SINGLE_CPU := 1
PROGRAMMABLE_RESET_ADDRESS := 1
+ENABLE_SVE_FOR_NS := 0
# Process flags
$(eval $(call add_define,HIKEY960_TSP_RAM_LOCATION_ID))
diff --git a/plat/hisilicon/poplar/platform.mk b/plat/hisilicon/poplar/platform.mk
index fc75ff3..28e0d1f 100644
--- a/plat/hisilicon/poplar/platform.mk
+++ b/plat/hisilicon/poplar/platform.mk
@@ -13,6 +13,7 @@
ERRATA_A53_855873 := 1
ERRATA_A53_835769 := 1
ERRATA_A53_843419 := 1
+ENABLE_SVE_FOR_NS := 0
ARM_GIC_ARCH := 2
$(eval $(call add_define,ARM_GIC_ARCH))
@@ -69,4 +70,3 @@
plat/hisilicon/poplar/bl31_plat_setup.c \
plat/hisilicon/poplar/plat_topology.c \
plat/hisilicon/poplar/plat_pm.c
-
diff --git a/plat/mediatek/mt6795/platform.mk b/plat/mediatek/mt6795/platform.mk
index 4ebc78e..8230067 100644
--- a/plat/mediatek/mt6795/platform.mk
+++ b/plat/mediatek/mt6795/platform.mk
@@ -66,3 +66,5 @@
$(eval $(call add_define,MTK_SIP_KERNEL_BOOT_ENABLE))
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/mediatek/mt8173/platform.mk b/plat/mediatek/mt8173/platform.mk
index cd01645..2eef81b 100644
--- a/plat/mediatek/mt8173/platform.mk
+++ b/plat/mediatek/mt8173/platform.mk
@@ -70,3 +70,6 @@
PROGRAMMABLE_RESET_ADDRESS := 1
$(eval $(call add_define,MTK_SIP_SET_AUTHORIZED_SECURE_REG_ENABLE))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/nvidia/tegra/platform.mk b/plat/nvidia/tegra/platform.mk
index 9a9e79e..ad60620 100644
--- a/plat/nvidia/tegra/platform.mk
+++ b/plat/nvidia/tegra/platform.mk
@@ -29,6 +29,9 @@
# do not use coherent memory
USE_COHERENT_MEM := 0
+# do not enable SVE
+ENABLE_SVE_FOR_NS := 0
+
include plat/nvidia/tegra/common/tegra_common.mk
include ${SOC_DIR}/platform_${TARGET_SOC}.mk
diff --git a/plat/qemu/platform.mk b/plat/qemu/platform.mk
index 2a7415f..43ab846 100644
--- a/plat/qemu/platform.mk
+++ b/plat/qemu/platform.mk
@@ -153,3 +153,6 @@
# Process flags
$(eval $(call add_define,BL32_RAM_LOCATION_ID))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/rockchip/rk3328/platform.mk b/plat/rockchip/rk3328/platform.mk
index 8863fb4..5de4680 100644
--- a/plat/rockchip/rk3328/platform.mk
+++ b/plat/rockchip/rk3328/platform.mk
@@ -48,7 +48,10 @@
${RK_PLAT_SOC}/drivers/pmu/pmu.c \
${RK_PLAT_SOC}/drivers/soc/soc.c
-ENABLE_PLAT_COMPAT := 0
+ENABLE_PLAT_COMPAT := 0
$(eval $(call add_define,PLAT_EXTRA_LD_SCRIPT))
$(eval $(call add_define,PLAT_SKIP_OPTEE_S_EL1_INT_REGISTER))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/rockchip/rk3368/platform.mk b/plat/rockchip/rk3368/platform.mk
index f6960cf..d3c6eef 100644
--- a/plat/rockchip/rk3368/platform.mk
+++ b/plat/rockchip/rk3368/platform.mk
@@ -48,6 +48,9 @@
${RK_PLAT_SOC}/drivers/soc/soc.c \
${RK_PLAT_SOC}/drivers/ddr/ddr_rk3368.c \
-ENABLE_PLAT_COMPAT := 0
+ENABLE_PLAT_COMPAT := 0
$(eval $(call add_define,PLAT_EXTRA_LD_SCRIPT))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/rockchip/rk3399/platform.mk b/plat/rockchip/rk3399/platform.mk
index 6cd5b24..33b9723 100644
--- a/plat/rockchip/rk3399/platform.mk
+++ b/plat/rockchip/rk3399/platform.mk
@@ -92,3 +92,6 @@
.PHONY: $(RK3399M0FW)
$(RK3399M0FW): | ${BUILD_M0}
$(MAKE) -C ${RK_PLAT_SOC}/drivers/m0 BUILD=$(abspath ${BUILD_PLAT}/m0)
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
diff --git a/plat/socionext/uniphier/platform.mk b/plat/socionext/uniphier/platform.mk
index c91abb6..3c78054 100644
--- a/plat/socionext/uniphier/platform.mk
+++ b/plat/socionext/uniphier/platform.mk
@@ -10,6 +10,7 @@
override LOAD_IMAGE_V2 := 1
override USE_COHERENT_MEM := 1
override USE_TBBR_DEFS := 1
+override ENABLE_SVE_FOR_NS := 0
# Cortex-A53 revision r0p4-51rel0
# needed for LD20, unneeded for LD11, PXs3 (no ACE)
diff --git a/plat/xilinx/zynqmp/platform.mk b/plat/xilinx/zynqmp/platform.mk
index ca87cc8..cb3b442 100644
--- a/plat/xilinx/zynqmp/platform.mk
+++ b/plat/xilinx/zynqmp/platform.mk
@@ -11,6 +11,9 @@
SEPARATE_CODE_AND_RODATA := 1
override RESET_TO_BL31 := 1
+# Do not enable SVE
+ENABLE_SVE_FOR_NS := 0
+
ifdef ZYNQMP_ATF_MEM_BASE
$(eval $(call add_define,ZYNQMP_ATF_MEM_BASE))