feat(d128): add support for FEAT_D128
This patch disables trapping to EL3 when the FEAT_D128
specific registers are accessed by setting the SCR_EL3.D128En bit.
If FEAT_D128 is implemented, then FEAT_SYSREG128 is implemented.
With FEAT_SYSREG128 certain system registers are treated as 128-bit,
so we should be context saving and restoring 128-bits instead of 64-bit
when FEAT_D128 is enabled.
FEAT_SYSREG128 adds support for MRRS and MSRR instruction which
helps us to read write to 128-bit system register.
Refer to Arm Architecture Manual for further details.
Change the FVP platform to default to handling this as a dynamic option
so the right decision can be made by the code at runtime.
Change-Id: I1a53db5eac29e56c8fbdcd4961ede3abfcb2411a
Signed-off-by: Jayanth Dodderi Chidanand <jayanthdodderi.chidanand@arm.com>
Signed-off-by: Govindraj Raja <govindraj.raja@arm.com>
diff --git a/include/arch/aarch32/arch.h b/include/arch/aarch32/arch.h
index d32ead4..d2591dd 100644
--- a/include/arch/aarch32/arch.h
+++ b/include/arch/aarch32/arch.h
@@ -697,8 +697,7 @@
/* PAR fields */
#define PAR_F_SHIFT U(0)
#define PAR_F_MASK ULL(0x1)
-#define PAR_ADDR_SHIFT U(12)
-#define PAR_ADDR_MASK (BIT_64(40) - ULL(1)) /* 40-bits-wide page address */
+#define PAR_ADDR_MASK GENMASK_64(39, 12) /* 28-bits-wide page address */
/*******************************************************************************
* Definitions for system register interface to AMU for FEAT_AMUv1
diff --git a/include/arch/aarch32/arch_features.h b/include/arch/aarch32/arch_features.h
index abe34a4..a29b672 100644
--- a/include/arch/aarch32/arch_features.h
+++ b/include/arch/aarch32/arch_features.h
@@ -194,5 +194,7 @@
static inline bool is_feat_ebep_present(void) { return false; }
__attribute__((always_inline))
static inline bool is_feat_sebep_present(void) { return false; }
+__attribute__((always_inline))
+static inline bool is_feat_d128_present(void) { return false; }
#endif /* ARCH_FEATURES_H */
diff --git a/include/arch/aarch64/arch.h b/include/arch/aarch64/arch.h
index 6a19822..3f0120c 100644
--- a/include/arch/aarch64/arch.h
+++ b/include/arch/aarch64/arch.h
@@ -331,6 +331,7 @@
#define PARANGE_0100 U(44)
#define PARANGE_0101 U(48)
#define PARANGE_0110 U(52)
+#define PARANGE_0111 U(56)
#define ID_AA64MMFR0_EL1_ECV_SHIFT U(60)
#define ID_AA64MMFR0_EL1_ECV_MASK ULL(0xf)
@@ -394,6 +395,10 @@
/* ID_AA64MMFR3_EL1 definitions */
#define ID_AA64MMFR3_EL1 S3_0_C0_C7_3
+#define ID_AA64MMFR3_EL1_D128_SHIFT U(32)
+#define ID_AA64MMFR3_EL1_D128_MASK ULL(0xf)
+#define D128_IMPLEMENTED ULL(0x1)
+
#define ID_AA64MMFR3_EL1_S2POE_SHIFT U(20)
#define ID_AA64MMFR3_EL1_S2POE_MASK ULL(0xf)
@@ -594,6 +599,7 @@
#define SCR_FGTEN2_BIT (UL(1) << 59)
#define SCR_NSE_BIT (ULL(1) << SCR_NSE_SHIFT)
#define SCR_GPF_BIT (UL(1) << 48)
+#define SCR_D128En_BIT (UL(1) << 47)
#define SCR_TWEDEL_SHIFT U(30)
#define SCR_TWEDEL_MASK ULL(0xf)
#define SCR_PIEN_BIT (UL(1) << 45)
@@ -1179,8 +1185,9 @@
/* PAR_EL1 fields */
#define PAR_F_SHIFT U(0)
#define PAR_F_MASK ULL(0x1)
-#define PAR_ADDR_SHIFT U(12)
-#define PAR_ADDR_MASK (BIT(40) - ULL(1)) /* 40-bits-wide page address */
+
+#define PAR_D128_ADDR_MASK GENMASK(55, 12) /* 44-bits-wide page address */
+#define PAR_ADDR_MASK GENMASK(51, 12) /* 40-bits-wide page address */
/*******************************************************************************
* Definitions for system register interface to SPE
diff --git a/include/arch/aarch64/arch_features.h b/include/arch/aarch64/arch_features.h
index de21fea..ec38d76 100644
--- a/include/arch/aarch64/arch_features.h
+++ b/include/arch/aarch64/arch_features.h
@@ -138,6 +138,8 @@
* +----------------------------+
* | FEAT_SCTLR2 |
* +----------------------------+
+ * | FEAT_D128 |
+ * +----------------------------+
*/
__attribute__((always_inline))
@@ -275,6 +277,11 @@
ID_AA64MMFR3_EL1_SCTLR2_MASK, SCTLR2_IMPLEMENTED,
ENABLE_FEAT_SCTLR2)
+/* FEAT_D128 */
+CREATE_FEATURE_FUNCS(feat_d128, id_aa64mmfr3_el1, ID_AA64MMFR3_EL1_D128_SHIFT,
+ ID_AA64MMFR3_EL1_D128_MASK, D128_IMPLEMENTED,
+ ENABLE_FEAT_D128)
+
__attribute__((always_inline))
static inline bool is_feat_sxpie_supported(void)
{
diff --git a/include/arch/aarch64/arch_helpers.h b/include/arch/aarch64/arch_helpers.h
index acaa1b8..9c36e4b 100644
--- a/include/arch/aarch64/arch_helpers.h
+++ b/include/arch/aarch64/arch_helpers.h
@@ -13,6 +13,7 @@
#include <string.h>
#include <arch.h>
+#include <lib/extensions/sysreg128.h>
/**********************************************************************
* Macros which create inline functions to read or write CPU system
@@ -263,7 +264,12 @@
#define write_daifclr(val) SYSREG_WRITE_CONST(daifclr, val)
#define write_daifset(val) SYSREG_WRITE_CONST(daifset, val)
+#if ENABLE_FEAT_D128
+DECLARE_SYSREG128_RW_FUNCS(par_el1)
+#else
DEFINE_SYSREG_RW_FUNCS(par_el1)
+#endif
+
DEFINE_IDREG_READ_FUNC(id_pfr1_el1)
DEFINE_IDREG_READ_FUNC(id_aa64isar0_el1)
DEFINE_IDREG_READ_FUNC(id_aa64isar1_el1)
@@ -443,13 +449,21 @@
DEFINE_SYSREG_RW_FUNCS(tcr_el2)
DEFINE_SYSREG_RW_FUNCS(tcr_el3)
+#if ENABLE_FEAT_D128
+DECLARE_SYSREG128_RW_FUNCS(ttbr0_el1)
+DECLARE_SYSREG128_RW_FUNCS(ttbr1_el1)
+DECLARE_SYSREG128_RW_FUNCS(ttbr0_el2)
+DECLARE_SYSREG128_RW_FUNCS(ttbr1_el2)
+DECLARE_SYSREG128_RW_FUNCS(vttbr_el2)
+#else
DEFINE_SYSREG_RW_FUNCS(ttbr0_el1)
-DEFINE_SYSREG_RW_FUNCS(ttbr0_el2)
-DEFINE_SYSREG_RW_FUNCS(ttbr0_el3)
-
DEFINE_SYSREG_RW_FUNCS(ttbr1_el1)
-
+DEFINE_SYSREG_RW_FUNCS(ttbr0_el2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ttbr1_el2, TTBR1_EL2)
DEFINE_SYSREG_RW_FUNCS(vttbr_el2)
+#endif
+
+DEFINE_SYSREG_RW_FUNCS(ttbr0_el3)
DEFINE_SYSREG_RW_FUNCS(cptr_el2)
DEFINE_SYSREG_RW_FUNCS(cptr_el3)
@@ -574,7 +588,6 @@
/* Armv8.1 VHE Registers */
DEFINE_RENAME_SYSREG_RW_FUNCS(contextidr_el2, CONTEXTIDR_EL2)
-DEFINE_RENAME_SYSREG_RW_FUNCS(ttbr1_el2, TTBR1_EL2)
/* Armv8.2 ID Registers */
DEFINE_RENAME_IDREG_READ_FUNC(id_aa64mmfr2_el1, ID_AA64MMFR2_EL1)
@@ -671,8 +684,13 @@
DEFINE_RENAME_SYSREG_RW_FUNCS(gcspr_el0, GCSPR_EL0)
/* FEAT_THE Registers */
+#if ENABLE_FEAT_D128
+DECLARE_SYSREG128_RW_FUNCS(rcwmask_el1)
+DECLARE_SYSREG128_RW_FUNCS(rcwsmask_el1)
+#else
DEFINE_RENAME_SYSREG_RW_FUNCS(rcwmask_el1, RCWMASK_EL1)
DEFINE_RENAME_SYSREG_RW_FUNCS(rcwsmask_el1, RCWSMASK_EL1)
+#endif
/* FEAT_SCTLR2 Registers */
DEFINE_RENAME_SYSREG_RW_FUNCS(sctlr2_el1, SCTLR2_EL1)
diff --git a/include/common/par.h b/include/common/par.h
new file mode 100644
index 0000000..c8d67a3
--- /dev/null
+++ b/include/common/par.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2024, Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef PAR_H
+#define PAR_H
+
+#include<arch_features.h>
+#include<lib/extensions/sysreg128.h>
+
+static inline uint64_t get_par_el1_pa(sysreg_t par)
+{
+ uint64_t pa = par & UINT64_MAX;
+ /* PA, bits [51:12] is Output address */
+ uint64_t mask = PAR_ADDR_MASK;
+
+#if ENABLE_FEAT_D128
+ /* If D128 is in use, the PA is in the upper 64-bit word of PAR_EL1 */
+ if (is_feat_d128_supported() && (par & PAR_EL1_D128)) {
+ pa = (par >> 64) & UINT64_MAX;
+ /* PA, bits [55:12] is Output address */
+ mask = PAR_D128_ADDR_MASK;
+ }
+#endif
+ return pa & mask;
+}
+
+#endif /* PAR_H */
diff --git a/include/lib/el3_runtime/context_el1.h b/include/lib/el3_runtime/context_el1.h
index 94af210..4379bcf 100644
--- a/include/lib/el3_runtime/context_el1.h
+++ b/include/lib/el3_runtime/context_el1.h
@@ -7,6 +7,8 @@
#ifndef CONTEXT_EL1_H
#define CONTEXT_EL1_H
+#include <lib/extensions/sysreg128.h>
+
#ifndef __ASSEMBLER__
/*******************************************************************************
@@ -28,15 +30,12 @@
uint64_t csselr_el1;
uint64_t sp_el1;
uint64_t esr_el1;
- uint64_t ttbr0_el1;
- uint64_t ttbr1_el1;
uint64_t mair_el1;
uint64_t amair_el1;
uint64_t actlr_el1;
uint64_t tpidr_el1;
uint64_t tpidr_el0;
uint64_t tpidrro_el0;
- uint64_t par_el1;
uint64_t far_el1;
uint64_t afsr0_el1;
uint64_t afsr1_el1;
@@ -44,6 +43,9 @@
uint64_t vbar_el1;
uint64_t mdccint_el1;
uint64_t mdscr_el1;
+ sysreg_t par_el1;
+ sysreg_t ttbr0_el1;
+ sysreg_t ttbr1_el1;
} el1_common_regs_t;
typedef struct el1_aarch32_regs {
@@ -108,8 +110,8 @@
} el1_gcs_regs_t;
typedef struct el1_the_regs {
- uint64_t rcwmask_el1;
- uint64_t rcwsmask_el1;
+ sysreg_t rcwmask_el1;
+ sysreg_t rcwsmask_el1;
} el1_the_regs_t;
typedef struct el1_sctlr2_regs {
diff --git a/include/lib/el3_runtime/context_el2.h b/include/lib/el3_runtime/context_el2.h
index ad0b68f..7374e39 100644
--- a/include/lib/el3_runtime/context_el2.h
+++ b/include/lib/el3_runtime/context_el2.h
@@ -7,7 +7,10 @@
#ifndef CONTEXT_EL2_H
#define CONTEXT_EL2_H
+#include <lib/extensions/sysreg128.h>
+
#ifndef __ASSEMBLER__
+
/*******************************************************************************
* EL2 Registers:
* AArch64 EL2 system register context structure for preserving the
@@ -40,12 +43,12 @@
uint64_t sp_el2;
uint64_t tcr_el2;
uint64_t tpidr_el2;
- uint64_t ttbr0_el2;
uint64_t vbar_el2;
uint64_t vmpidr_el2;
uint64_t vpidr_el2;
uint64_t vtcr_el2;
- uint64_t vttbr_el2;
+ sysreg_t vttbr_el2;
+ sysreg_t ttbr0_el2;
} el2_common_regs_t;
typedef struct el2_mte2_regs {
@@ -75,7 +78,7 @@
typedef struct el2_vhe_regs {
uint64_t contextidr_el2;
- uint64_t ttbr1_el2;
+ sysreg_t ttbr1_el2;
} el2_vhe_regs_t;
typedef struct el2_ras_regs {
@@ -222,6 +225,9 @@
#define write_el2_ctx_common(ctx, reg, val) ((((ctx)->common).reg) \
= (uint64_t) (val))
+#define write_el2_ctx_sysreg128(ctx, reg, val) ((((ctx)->common).reg) \
+ = (sysreg_t) (val))
+
#if ENABLE_FEAT_MTE2
#define read_el2_ctx_mte2(ctx, reg) (((ctx)->mte2).reg)
#define write_el2_ctx_mte2(ctx, reg, val) ((((ctx)->mte2).reg) \
@@ -262,6 +268,9 @@
#define read_el2_ctx_vhe(ctx, reg) (((ctx)->vhe).reg)
#define write_el2_ctx_vhe(ctx, reg, val) ((((ctx)->vhe).reg) \
= (uint64_t) (val))
+
+#define write_el2_ctx_vhe_sysreg128(ctx, reg, val) ((((ctx)->vhe).reg) \
+ = (sysreg_t) (val))
#else
#define read_el2_ctx_vhe(ctx, reg) ULL(0)
#define write_el2_ctx_vhe(ctx, reg, val)
diff --git a/include/lib/extensions/sysreg128.h b/include/lib/extensions/sysreg128.h
new file mode 100644
index 0000000..8854856
--- /dev/null
+++ b/include/lib/extensions/sysreg128.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2024, Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef SYSREG128_H
+#define SYSREG128_H
+
+#ifndef __ASSEMBLER__
+
+#if ENABLE_FEAT_D128
+#include <stdint.h>
+
+typedef uint128_t sysreg_t;
+
+#define PAR_EL1_D128 (((sysreg_t)(1ULL)) << (64))
+
+#define _DECLARE_SYSREG128_READ_FUNC(_name) \
+uint128_t read_ ## _name(void);
+
+#define _DECLARE_SYSREG128_WRITE_FUNC(_name) \
+void write_ ## _name(uint128_t v);
+
+#define DECLARE_SYSREG128_RW_FUNCS(_name) \
+ _DECLARE_SYSREG128_READ_FUNC(_name) \
+ _DECLARE_SYSREG128_WRITE_FUNC(_name)
+#else
+
+typedef uint64_t sysreg_t;
+
+#endif /* ENABLE_FEAT_D128 */
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* SYSREG128_H */