Merge "fix(arm): load dt before updating entry point" into integration
diff --git a/Makefile b/Makefile
index f736a3b..b0363cb 100644
--- a/Makefile
+++ b/Makefile
@@ -256,10 +256,12 @@
 				-Wlogical-op
 
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105523
-TF_CFLAGS		+= 	$(call cc_option, --param=min-pagesize=0)
+TF_CFLAGS_MIN_PAGE_SIZE	:=	$(call cc_option, --param=min-pagesize=0)
+TF_CFLAGS		+=	$(TF_CFLAGS_MIN_PAGE_SIZE)
 
 ifeq ($(HARDEN_SLS), 1)
-        TF_CFLAGS_aarch64       +=      $(call cc_option, -mharden-sls=all)
+        TF_CFLAGS_MHARDEN_SLS	:=      $(call cc_option, -mharden-sls=all)
+        TF_CFLAGS_aarch64	+=      $(TF_CFLAGS_MHARDEN_SLS)
 endif
 
 else
@@ -1189,6 +1191,7 @@
 	SEPARATE_CODE_AND_RODATA \
 	SEPARATE_BL2_NOLOAD_REGION \
 	SEPARATE_NOBITS_REGION \
+	SEPARATE_RWDATA_REGION \
 	SEPARATE_SIMD_SECTION \
 	SPIN_ON_BL1_EXIT \
 	SPM_MM \
@@ -1257,6 +1260,7 @@
 	ENABLE_FEAT_FGT \
 	ENABLE_FEAT_FGT2 \
 	ENABLE_FEAT_HCX \
+	ENABLE_FEAT_LS64_ACCDATA \
 	ENABLE_FEAT_MTE2 \
 	ENABLE_FEAT_PAN \
 	ENABLE_FEAT_RNG \
@@ -1367,6 +1371,7 @@
 	SEPARATE_CODE_AND_RODATA \
 	SEPARATE_BL2_NOLOAD_REGION \
 	SEPARATE_NOBITS_REGION \
+	SEPARATE_RWDATA_REGION \
 	SEPARATE_SIMD_SECTION \
 	RECLAIM_INIT_CODE \
 	SPD_${SPD} \
@@ -1421,6 +1426,7 @@
 	ENABLE_FEAT_VHE \
 	ENABLE_FEAT_CSV2_2 \
 	ENABLE_FEAT_CSV2_3 \
+	ENABLE_FEAT_LS64_ACCDATA \
 	ENABLE_FEAT_PAN \
 	ENABLE_FEAT_TCR2 \
 	ENABLE_FEAT_THE \
diff --git a/bl31/bl31.ld.S b/bl31/bl31.ld.S
index 8698dff..867dedb 100644
--- a/bl31/bl31.ld.S
+++ b/bl31/bl31.ld.S
@@ -19,6 +19,12 @@
 #else /* SEPARATE_NOBITS_REGION */
 #   define NOBITS RAM
 #endif /* SEPARATE_NOBITS_REGION */
+
+#if SEPARATE_RWDATA_REGION
+    RAM_RW (rw): ORIGIN = BL31_RWDATA_BASE, LENGTH = BL31_RWDATA_LIMIT - BL31_RWDATA_BASE
+#else /* SEPARATE_RWDATA_REGION */
+#define RAM_RW RAM
+#endif /* SEPARATE_RWDATA_REGION */
 }
 
 #ifdef PLAT_EXTRA_LD_SCRIPT
@@ -136,10 +142,36 @@
     . = LOADADDR(.spm_shim_exceptions) + SIZEOF(.spm_shim_exceptions);
 #endif /* SPM_MM || (SPMC_AT_EL3 && SPMC_AT_EL3_SEL0_SP) */
 
+#if SEPARATE_RWDATA_REGION
+    . = BL31_RWDATA_BASE;
+    ASSERT(BL31_RWDATA_BASE == ALIGN(PAGE_SIZE),
+           "BL31_RWDATA_BASE address is not aligned on a page boundary.")
+
+    /*
+     * Define a linker symbol to mark the start of the RW memory area for this
+     * image.
+     */
+    __RW_START__ = . ;
+
+    DATA_SECTION >RAM_RW AT>RAM
+    __DATA_RAM_START__ = __DATA_START__;
+    __DATA_RAM_END__ = __DATA_END__;
+    __DATA_ROM_START__ = LOADADDR(.data);
+
+    . = ALIGN(PAGE_SIZE);
+    __RW_END__ = .;
+
-    __RW_START__ = .;
+    RELA_SECTION >RAM
+#else /* SEPARATE_RWDATA_REGION */
+    /*
+     * Define a linker symbol to mark the start of the RW memory area for this
+     * image.
+     */
+    __RW_START__ = . ;
 
     DATA_SECTION >RAM
     RELA_SECTION >RAM
+#endif /* SEPARATE_RWDATA_REGION */
 
 #ifdef BL31_PROGBITS_LIMIT
     ASSERT(
@@ -151,7 +183,9 @@
 #if SEPARATE_NOBITS_REGION
     . = ALIGN(PAGE_SIZE);
 
+#if !SEPARATE_RWDATA_REGION
     __RW_END__ = .;
+#endif /* SEPARATE_RWDATA_REGION */
     __BL31_END__ = .;
 
     ASSERT(. <= BL31_LIMIT, "BL31 image has exceeded its limit.")
@@ -203,7 +237,13 @@
 
     ASSERT(. <= BL31_NOBITS_LIMIT, "BL31 NOBITS region has exceeded its limit.")
 #else /* SEPARATE_NOBITS_REGION */
+    /*
+     * Define a linker symbol to mark the end of the RW memory area for this
+     * image.
+     */
+#if !SEPARATE_RWDATA_REGION
     __RW_END__ = .;
+#endif /* SEPARATE_RWDATA_REGION */
     __BL31_END__ = .;
 
     ASSERT(. <= BL31_LIMIT, "BL31 image has exceeded its limit.")
diff --git a/common/feat_detect.c b/common/feat_detect.c
index 6aa5e2e..8c03ab8 100644
--- a/common/feat_detect.c
+++ b/common/feat_detect.c
@@ -192,6 +192,11 @@
 	return ISOLATE_FIELD(read_id_aa64mmfr1_el1(), ID_AA64MMFR1_EL1_HCX_SHIFT,
 			     ID_AA64MMFR1_EL1_HCX_MASK);
 }
+static unsigned int read_feat_ls64_id_field(void)
+{
+	return ISOLATE_FIELD(read_id_aa64isar1_el1(), ID_AA64ISAR1_LS64_SHIFT,
+			     ID_AA64ISAR1_LS64_MASK);
+}
 static unsigned int read_feat_tcr2_id_field(void)
 {
 	return ISOLATE_FIELD(read_id_aa64mmfr3_el1(), ID_AA64MMFR3_EL1_TCRX_SHIFT,
@@ -367,6 +372,7 @@
 
 	/* v8.7 features */
 	check_feature(ENABLE_FEAT_HCX, read_feat_hcx_id_field(), "HCX", 1, 1);
+	check_feature(ENABLE_FEAT_LS64_ACCDATA, read_feat_ls64_id_field(), "LS64", 1, 3);
 
 	/* v8.9 features */
 	check_feature(ENABLE_FEAT_TCR2, read_feat_tcr2_id_field(),
diff --git a/docs/Makefile b/docs/Makefile
index 9fd7d76..68c0958 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -24,4 +24,5 @@
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 .DEFAULT: Makefile
+	$(if $(host-poetry),$(q)poetry -q install --with=docs)
 	$(q)$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/getting_started/build-options.rst b/docs/getting_started/build-options.rst
index cd10cd6..ab0b94d 100644
--- a/docs/getting_started/build-options.rst
+++ b/docs/getting_started/build-options.rst
@@ -499,6 +499,11 @@
    The flag is automatically disabled when the target
    architecture is AArch32.
 
+-  ``ENABLE_FEAT_LS64_ACCDATA``: Numeric value to enable access and save and
+   restore the ACCDATA_EL1 system register, at EL2 and below. This flag can
+   take the values 0 to 2, to align  with the ``ENABLE_FEAT`` mechanism.
+   Default value is ``0``.
+
 -  ``ENABLE_MPMM``: Boolean option to enable support for the Maximum Power
    Mitigation Mechanism supported by certain Arm cores, which allows the SoC
    firmware to detect and limit high activity events to assist in SoC processor
diff --git a/docs/getting_started/docs-build.rst b/docs/getting_started/docs-build.rst
index 50fff57..54e29dd 100644
--- a/docs/getting_started/docs-build.rst
+++ b/docs/getting_started/docs-build.rst
@@ -37,28 +37,11 @@
 Building rendered documentation
 -------------------------------
 
-To install Python dependencies using Poetry:
+The documentation can be compiled into HTML-formatted pages from the project
+root directory by running:
 
 .. code:: shell
 
-    poetry install
-
-Poetry will create a new virtual environment and install all dependencies listed
-in ``pyproject.toml``. You can get information about this environment, such as
-its location and the Python version, with the command:
-
-.. code:: shell
-
-    poetry env info
-
-If you have already sourced a virtual environment, Poetry will respect this and
-install dependencies there.
-
-Once all dependencies are installed, the documentation can be compiled into
-HTML-formatted pages from the project root directory by running:
-
-.. code:: shell
-
    poetry run make doc
 
 Output from the build process will be placed in: ``docs/build/html``.
@@ -129,7 +112,7 @@
         bash -c 'cd /tf-a &&
             apt-get update && apt-get install -y curl plantuml &&
             curl -sSL https://install.python-poetry.org | python3 - &&
-            ~/.local/bin/poetry install && ~/.local/bin/poetry run make doc'
+            ~/.local/bin/poetry run make doc'
 
 The above command fetches the ``sphinxdoc/sphinx`` container from `docker
 hub`_, launches the container, installs documentation requirements and finally
@@ -138,7 +121,7 @@
 
 --------------
 
-*Copyright (c) 2019-2023, Arm Limited. All rights reserved.*
+*Copyright (c) 2019-2024, Arm Limited. All rights reserved.*
 
 .. _Sphinx: http://www.sphinx-doc.org/en/master/
 .. _Poetry: https://python-poetry.org/docs/
diff --git a/drivers/auth/mbedtls/mbedtls_psa_crypto.c b/drivers/auth/mbedtls/mbedtls_psa_crypto.c
index f2ccf15..0e4b57e 100644
--- a/drivers/auth/mbedtls/mbedtls_psa_crypto.c
+++ b/drivers/auth/mbedtls/mbedtls_psa_crypto.c
@@ -27,8 +27,10 @@
 
 #define LIB_NAME		"mbed TLS PSA"
 
-/* Maximum length of R_S pair in the ECDSA signature in bytes */
-#define MAX_ECDSA_R_S_PAIR_LEN	64U
+/* Minimum required size for a buffer containing a raw EC signature when using
+ * a maximum curve size of 384 bits.
+ * This is calculated as 2 * (384 / 8). */
+#define ECDSA_SIG_BUFFER_SIZE	96U
 
 /* Size of ASN.1 length and tag in bytes*/
 #define SIZE_OF_ASN1_LEN	1U
@@ -199,7 +201,7 @@
 	psa_key_id_t psa_key_id;
 	mbedtls_pk_type_t pk_alg;
 	psa_algorithm_t psa_alg;
-	__unused unsigned char reformatted_sig[MAX_ECDSA_R_S_PAIR_LEN] = {0};
+	__unused unsigned char reformatted_sig[ECDSA_SIG_BUFFER_SIZE] = {0};
 	unsigned char *local_sig_ptr;
 	size_t local_sig_len;
 
@@ -252,7 +254,7 @@
 		size_t key_bits = psa_get_key_bits(&psa_key_attr);
 
 		rc = mbedtls_ecdsa_der_to_raw(key_bits, p, local_sig_len,
-					      reformatted_sig, MAX_ECDSA_R_S_PAIR_LEN,
+					      reformatted_sig, ECDSA_SIG_BUFFER_SIZE,
 					      &local_sig_len);
 		if (rc != 0) {
 			rc = CRYPTO_ERR_SIGNATURE;
diff --git a/include/arch/aarch32/arch_features.h b/include/arch/aarch32/arch_features.h
index a29b672..e347240 100644
--- a/include/arch/aarch32/arch_features.h
+++ b/include/arch/aarch32/arch_features.h
@@ -196,5 +196,7 @@
 static inline bool is_feat_sebep_present(void) { return false; }
 __attribute__((always_inline))
 static inline bool is_feat_d128_present(void) { return false; }
+__attribute__((always_inline))
+static inline bool is_feat_ls64_accdata_present(void) { return false; }
 
 #endif /* ARCH_FEATURES_H */
diff --git a/include/arch/aarch64/arch.h b/include/arch/aarch64/arch.h
index 3f0120c..737d07a 100644
--- a/include/arch/aarch64/arch.h
+++ b/include/arch/aarch64/arch.h
@@ -293,6 +293,18 @@
 /* ID_AA64ISAR1_EL1 definitions */
 #define ID_AA64ISAR1_EL1		S3_0_C0_C6_1
 
+#define ID_AA64ISAR1_LS64_SHIFT		U(60)
+#define ID_AA64ISAR1_LS64_MASK		ULL(0xf)
+#define LS64_ACCDATA_IMPLEMENTED	ULL(0x3)
+#define LS64_V_IMPLEMENTED		ULL(0x2)
+#define LS64_IMPLEMENTED		ULL(0x1)
+#define LS64_NOT_IMPLEMENTED		ULL(0x0)
+
+#define ID_AA64ISAR1_SB_SHIFT		U(36)
+#define ID_AA64ISAR1_SB_MASK		ULL(0xf)
+#define SB_IMPLEMENTED			ULL(0x1)
+#define SB_NOT_IMPLEMENTED		ULL(0x0)
+
 #define ID_AA64ISAR1_GPI_SHIFT		U(28)
 #define ID_AA64ISAR1_GPI_MASK		ULL(0xf)
 #define ID_AA64ISAR1_GPA_SHIFT		U(24)
@@ -303,11 +315,6 @@
 #define ID_AA64ISAR1_APA_SHIFT		U(4)
 #define ID_AA64ISAR1_APA_MASK		ULL(0xf)
 
-#define ID_AA64ISAR1_SB_SHIFT		U(36)
-#define ID_AA64ISAR1_SB_MASK		ULL(0xf)
-#define SB_IMPLEMENTED			ULL(0x1)
-#define SB_NOT_IMPLEMENTED		ULL(0x0)
-
 /* ID_AA64ISAR2_EL1 definitions */
 #define ID_AA64ISAR2_EL1		S3_0_C0_C6_2
 
@@ -606,11 +613,13 @@
 #define SCR_SCTLR2En_BIT	(UL(1) << 44)
 #define SCR_TCR2EN_BIT		(UL(1) << 43)
 #define SCR_RCWMASKEn_BIT	(UL(1) << 42)
+#define SCR_ENTP2_SHIFT		U(41)
+#define SCR_ENTP2_BIT		(UL(1) << SCR_ENTP2_SHIFT)
 #define SCR_TRNDR_BIT		(UL(1) << 40)
 #define SCR_GCSEn_BIT		(UL(1) << 39)
 #define SCR_HXEn_BIT		(UL(1) << 38)
-#define SCR_ENTP2_SHIFT		U(41)
-#define SCR_ENTP2_BIT		(UL(1) << SCR_ENTP2_SHIFT)
+#define SCR_ADEn_BIT		(UL(1) << 37)
+#define SCR_EnAS0_BIT		(UL(1) << 36)
 #define SCR_AMVOFFEN_SHIFT	U(35)
 #define SCR_AMVOFFEN_BIT	(UL(1) << SCR_AMVOFFEN_SHIFT)
 #define SCR_TWEDEn_BIT		(UL(1) << 29)
@@ -1504,6 +1513,11 @@
 #define SCTLR2_EL1		S3_0_C1_C0_3
 
 /*******************************************************************************
+ * FEAT_LS64_ACCDATA - LoadStore64B with status data
+ ******************************************************************************/
+#define ACCDATA_EL1		S3_0_C13_C0_5
+
+/*******************************************************************************
  * Definitions for DynamicIQ Shared Unit registers
  ******************************************************************************/
 #define CLUSTERPWRDN_EL1	S3_0_c15_c3_6
diff --git a/include/arch/aarch64/arch_features.h b/include/arch/aarch64/arch_features.h
index ec38d76..59188da 100644
--- a/include/arch/aarch64/arch_features.h
+++ b/include/arch/aarch64/arch_features.h
@@ -140,6 +140,8 @@
  * +----------------------------+
  * |	FEAT_D128		|
  * +----------------------------+
+ * |	FEAT_LS64_ACCDATA	|
+ * +----------------------------+
  */
 
 __attribute__((always_inline))
@@ -421,6 +423,11 @@
 CREATE_FEATURE_FUNCS(feat_sme2, id_aa64pfr1_el1, ID_AA64PFR1_EL1_SME_SHIFT,
 		     ID_AA64PFR1_EL1_SME_MASK, SME2_IMPLEMENTED, ENABLE_SME2_FOR_NS)
 
+/* FEAT_LS64_ACCDATA: */
+CREATE_FEATURE_FUNCS(feat_ls64_accdata, id_aa64isar1_el1, ID_AA64ISAR1_LS64_SHIFT,
+		     ID_AA64ISAR1_LS64_MASK, LS64_ACCDATA_IMPLEMENTED,
+		     ENABLE_FEAT_LS64_ACCDATA)
+
 /*******************************************************************************
  * Function to get hardware granularity support
  ******************************************************************************/
diff --git a/include/arch/aarch64/arch_helpers.h b/include/arch/aarch64/arch_helpers.h
index 9c36e4b..119c428 100644
--- a/include/arch/aarch64/arch_helpers.h
+++ b/include/arch/aarch64/arch_helpers.h
@@ -696,6 +696,9 @@
 DEFINE_RENAME_SYSREG_RW_FUNCS(sctlr2_el1, SCTLR2_EL1)
 DEFINE_RENAME_SYSREG_RW_FUNCS(sctlr2_el2, SCTLR2_EL2)
 
+/* FEAT_LS64_ACCDATA Registers */
+DEFINE_RENAME_SYSREG_RW_FUNCS(accdata_el1, ACCDATA_EL1)
+
 /* DynamIQ Control registers */
 DEFINE_RENAME_SYSREG_RW_FUNCS(clusterpwrdn_el1, CLUSTERPWRDN_EL1)
 DEFINE_RENAME_SYSREG_RW_FUNCS(clusterpmcr_el1, CLUSTERPMCR_EL1)
diff --git a/include/arch/aarch64/el3_common_macros.S b/include/arch/aarch64/el3_common_macros.S
index 1666e3b..204625c 100644
--- a/include/arch/aarch64/el3_common_macros.S
+++ b/include/arch/aarch64/el3_common_macros.S
@@ -340,7 +340,9 @@
 #endif
 
 #if defined(IMAGE_BL1) ||	\
-	(defined(IMAGE_BL2) && RESET_TO_BL2 && BL2_IN_XIP_MEM)
+	(defined(IMAGE_BL2) && RESET_TO_BL2 && BL2_IN_XIP_MEM) || \
+	(defined(IMAGE_BL31) && SEPARATE_RWDATA_REGION)
+
 		adrp	x0, __DATA_RAM_START__
 		add	x0, x0, :lo12:__DATA_RAM_START__
 		adrp	x1, __DATA_ROM_START__
diff --git a/include/lib/el3_runtime/context_el1.h b/include/lib/el3_runtime/context_el1.h
index 4379bcf..7bc0235 100644
--- a/include/lib/el3_runtime/context_el1.h
+++ b/include/lib/el3_runtime/context_el1.h
@@ -118,6 +118,10 @@
 	uint64_t sctlr2_el1;
 } el1_sctlr2_regs_t;
 
+typedef struct el1_ls64_regs {
+	uint64_t accdata_el1;
+} el1_ls64_regs_t;
+
 typedef struct el1_sysregs {
 
 	el1_common_regs_t common;
@@ -174,6 +178,9 @@
 	el1_sctlr2_regs_t sctlr2;
 #endif
 
+#if ENABLE_FEAT_LS64_ACCDATA
+	el1_ls64_regs_t ls64;
+#endif
 } el1_sysregs_t;
 
 
@@ -304,6 +311,14 @@
 #define write_el1_ctx_sctlr2(ctx, reg, val)
 #endif /* ENABLE_FEAT_SCTLR2 */
 
+#if ENABLE_FEAT_LS64_ACCDATA
+#define read_el1_ctx_ls64(ctx, reg)		(((ctx)->ls64).reg)
+#define write_el1_ctx_ls64(ctx, reg, val)	((((ctx)->ls64).reg)	\
+							= (uint64_t) (val))
+#else
+#define read_el1_ctx_ls64(ctx, reg)		ULL(0)
+#define write_el1_ctx_ls64(ctx, reg, val)
+#endif /* ENABLE_FEAT_LS64_ACCDATA */
 /******************************************************************************/
 #endif /* __ASSEMBLER__ */
 
diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c
index 8aa2ccc..4ae1306 100644
--- a/lib/el3_runtime/aarch64/context_mgmt.c
+++ b/lib/el3_runtime/aarch64/context_mgmt.c
@@ -427,6 +427,15 @@
 	}
 
 	/*
+	 * If FEAT_LS64_ACCDATA is enabled, enable access to ACCDATA_EL1 by
+	 * setting SCR_EL3.ADEn and allow the ST64BV0 instruction by setting
+	 * SCR_EL3.EnAS0.
+	 */
+	if (is_feat_ls64_accdata_supported()) {
+		scr_el3 |= SCR_ADEn_BIT | SCR_EnAS0_BIT;
+	}
+
+	/*
 	 * If FEAT_RNG_TRAP is enabled, all reads of the RNDR and RNDRRS
 	 * registers are trapped to EL3.
 	 */
@@ -1759,6 +1768,9 @@
 		write_el1_ctx_sctlr2(ctx, sctlr2_el1, read_sctlr2_el1());
 	}
 
+	if (is_feat_ls64_accdata_supported()) {
+		write_el1_ctx_ls64(ctx, accdata_el1, read_accdata_el1());
+	}
 }
 
 static void el1_sysregs_context_restore(el1_sysregs_t *ctx)
@@ -1864,6 +1876,9 @@
 		write_sctlr2_el1(read_el1_ctx_sctlr2(ctx, sctlr2_el1));
 	}
 
+	if (is_feat_ls64_accdata_supported()) {
+		write_accdata_el1(read_el1_ctx_ls64(ctx, accdata_el1));
+	}
 }
 
 /*******************************************************************************
diff --git a/lib/romlib/Makefile b/lib/romlib/Makefile
index 4cac75b..367487a 100644
--- a/lib/romlib/Makefile
+++ b/lib/romlib/Makefile
@@ -83,7 +83,7 @@
 	$(s)echo "  TBL     $@"
 	$(q)$(ROMLIB_GEN) gentbl --output $@ --bti=$(ENABLE_BTI) $<
 
-$(BUILD_DIR)/romlib.ldflags: ../../$(PLAT_DIR)/jmptbl.i
+$(BUILD_DIR)/romlib.ldflags: ../../$(PLAT_DIR)/jmptbl.i | $$(@D)/
 	$(s)echo "  LDFLAGS $@"
 	$(q)$(ROMLIB_GEN) link-flags $< > $@
 
diff --git a/make_helpers/arch_features.mk b/make_helpers/arch_features.mk
index 39f6223..d378a55 100644
--- a/make_helpers/arch_features.mk
+++ b/make_helpers/arch_features.mk
@@ -336,6 +336,9 @@
 # Flag to enable FEAT_FGT2 (Fine Granular Traps 2)
 ENABLE_FEAT_FGT2			?=	0
 
+# LoadStore64Bytes extension using the ACCDATA_EL1 system register
+ENABLE_FEAT_LS64_ACCDATA		?=	0
+
 #----
 # 8.8
 #----
diff --git a/make_helpers/build_macros.mk b/make_helpers/build_macros.mk
index f523074..d454efd 100644
--- a/make_helpers/build_macros.mk
+++ b/make_helpers/build_macros.mk
@@ -96,6 +96,10 @@
 
 # Convenience function to check for a given compiler option. A call to
 # $(call cc_option, --no-XYZ) will return --no-XYZ if supported by the compiler
+# NOTE: consider assigning to an immediately expanded temporary variable before
+# assigning. This is because variables like TF_CFLAGS are recursively expanded
+# and assigning this directly will cause it to be expanded every time the
+# variable is used, potentially thrashing multicore performance.
 define cc_option
 	$(shell if $($(ARCH)-cc) $(1) -c -x c /dev/null -o /dev/null >/dev/null 2>&1; then echo $(1); fi )
 endef
@@ -183,7 +187,7 @@
 
 define TOOL_ADD_IMG_PAYLOAD
 
-$(eval PRE_TOOL_FILTER := $($(call uppercase,$(1))_PRE_TOOL_FILTER))
+$(eval PRE_TOOL_FILTER := $($(1)_PRE_TOOL_FILTER))
 
 ifneq ($(PRE_TOOL_FILTER),)
 
@@ -220,7 +224,8 @@
 define TOOL_ADD_IMG
     # Build option to specify the image filename (SCP_BL2, BL33, etc)
     # This is the uppercase form of the first parameter
-    $(eval _V := $(call uppercase,$(1)))
+    $(eval BL := $(call uppercase,$(1)))
+    $(eval _V := $(BL))
 
     # $(check_$(1)_cmd) variable is executed in the check_$(1) target and also
     # is put into the ${CHECK_$(3)FIP_CMD} variable which is executed by the
@@ -235,10 +240,10 @@
 ifeq ($(4),1)
     $(eval ENC_BIN := ${BUILD_PLAT}/$(1)_enc.bin)
     $(call ENCRYPT_FW,$(value $(_V)),$(ENC_BIN))
-    $(call TOOL_ADD_IMG_PAYLOAD,$(1),$(value $(_V)),$(2),$(ENC_BIN),$(3), \
+    $(call TOOL_ADD_IMG_PAYLOAD,$(BL),$(value $(_V)),$(2),$(ENC_BIN),$(3), \
 		$(ENC_BIN))
 else
-    $(call TOOL_ADD_IMG_PAYLOAD,$(1),$(value $(_V)),$(2),$(if $(wildcard $(value $(_V))),$(value $(_V)),FORCE),$(3))
+    $(call TOOL_ADD_IMG_PAYLOAD,$(BL),$(value $(_V)),$(2),$(if $(wildcard $(value $(_V))),$(value $(_V)),FORCE),$(3))
 endif
 
 .PHONY: check_$(1)
@@ -284,10 +289,11 @@
 #   $(1) = output directory
 #   $(2) = source file (%.c)
 #   $(3) = library name
+#   $(4) = uppercase name of the library
 define MAKE_C_LIB
 $(eval OBJ := $(1)/$(patsubst %.c,%.o,$(notdir $(2))))
 $(eval DEP := $(patsubst %.o,%.d,$(OBJ)))
-$(eval LIB := $(call uppercase, $(notdir $(1))))
+$(eval LIB := $(notdir $(1)))
 
 $(OBJ): $(2) $(filter-out %.d,$(MAKEFILE_LIST)) | $$$$(@D)/
 	$$(s)echo "  CC      $$<"
@@ -301,6 +307,7 @@
 #   $(1) = output directory
 #   $(2) = source file (%.S)
 #   $(3) = library name
+#   $(4) = uppercase name of the library
 define MAKE_S_LIB
 $(eval OBJ := $(1)/$(patsubst %.S,%.o,$(notdir $(2))))
 $(eval DEP := $(patsubst %.o,%.d,$(OBJ)))
@@ -318,15 +325,16 @@
 #   $(1) = output directory
 #   $(2) = source file (%.c)
 #   $(3) = BL stage
+#   $(4) = uppercase BL stage
 define MAKE_C
 
 $(eval OBJ := $(1)/$(patsubst %.c,%.o,$(notdir $(2))))
 $(eval DEP := $(patsubst %.o,%.d,$(OBJ)))
 
-$(eval BL_DEFINES := IMAGE_$(call uppercase,$(3)) $($(call uppercase,$(3))_DEFINES) $(PLAT_BL_COMMON_DEFINES))
-$(eval BL_INCLUDE_DIRS := $($(call uppercase,$(3))_INCLUDE_DIRS) $(PLAT_BL_COMMON_INCLUDE_DIRS))
-$(eval BL_CPPFLAGS := $($(call uppercase,$(3))_CPPFLAGS) $(addprefix -D,$(BL_DEFINES)) $(addprefix -I,$(BL_INCLUDE_DIRS)) $(PLAT_BL_COMMON_CPPFLAGS))
-$(eval BL_CFLAGS := $($(call uppercase,$(3))_CFLAGS) $(PLAT_BL_COMMON_CFLAGS))
+$(eval BL_DEFINES := IMAGE_$(4) $($(4)_DEFINES) $(PLAT_BL_COMMON_DEFINES))
+$(eval BL_INCLUDE_DIRS := $($(4)_INCLUDE_DIRS) $(PLAT_BL_COMMON_INCLUDE_DIRS))
+$(eval BL_CPPFLAGS := $($(4)_CPPFLAGS) $(addprefix -D,$(BL_DEFINES)) $(addprefix -I,$(BL_INCLUDE_DIRS)) $(PLAT_BL_COMMON_CPPFLAGS))
+$(eval BL_CFLAGS := $($(4)_CFLAGS) $(PLAT_BL_COMMON_CFLAGS))
 
 $(OBJ): $(2) $(filter-out %.d,$(MAKEFILE_LIST)) | $$$$(@D)/
 	$$(s)echo "  CC      $$<"
@@ -341,15 +349,16 @@
 #   $(1) = output directory
 #   $(2) = assembly file (%.S)
 #   $(3) = BL stage
+#   $(4) = uppercase BL stage
 define MAKE_S
 
 $(eval OBJ := $(1)/$(patsubst %.S,%.o,$(notdir $(2))))
 $(eval DEP := $(patsubst %.o,%.d,$(OBJ)))
 
-$(eval BL_DEFINES := IMAGE_$(call uppercase,$(3)) $($(call uppercase,$(3))_DEFINES) $(PLAT_BL_COMMON_DEFINES))
-$(eval BL_INCLUDE_DIRS := $($(call uppercase,$(3))_INCLUDE_DIRS) $(PLAT_BL_COMMON_INCLUDE_DIRS))
-$(eval BL_CPPFLAGS := $($(call uppercase,$(3))_CPPFLAGS) $(addprefix -D,$(BL_DEFINES)) $(addprefix -I,$(BL_INCLUDE_DIRS)) $(PLAT_BL_COMMON_CPPFLAGS))
-$(eval BL_ASFLAGS := $($(call uppercase,$(3))_ASFLAGS) $(PLAT_BL_COMMON_ASFLAGS))
+$(eval BL_DEFINES := IMAGE_$(4) $($(4)_DEFINES) $(PLAT_BL_COMMON_DEFINES))
+$(eval BL_INCLUDE_DIRS := $($(4)_INCLUDE_DIRS) $(PLAT_BL_COMMON_INCLUDE_DIRS))
+$(eval BL_CPPFLAGS := $($(4)_CPPFLAGS) $(addprefix -D,$(BL_DEFINES)) $(addprefix -I,$(BL_INCLUDE_DIRS)) $(PLAT_BL_COMMON_CPPFLAGS))
+$(eval BL_ASFLAGS := $($(4)_ASFLAGS) $(PLAT_BL_COMMON_ASFLAGS))
 
 $(OBJ): $(2) $(filter-out %.d,$(MAKEFILE_LIST)) | $$$$(@D)/
 	$$(s)echo "  AS      $$<"
@@ -364,13 +373,14 @@
 #   $(1) = output linker script
 #   $(2) = input template
 #   $(3) = BL stage
+#   $(4) = uppercase BL stage
 define MAKE_LD
 
 $(eval DEP := $(1).d)
 
-$(eval BL_DEFINES := IMAGE_$(call uppercase,$(3)) $($(call uppercase,$(3))_DEFINES) $(PLAT_BL_COMMON_DEFINES))
-$(eval BL_INCLUDE_DIRS := $($(call uppercase,$(3))_INCLUDE_DIRS) $(PLAT_BL_COMMON_INCLUDE_DIRS))
-$(eval BL_CPPFLAGS := $($(call uppercase,$(3))_CPPFLAGS) $(addprefix -D,$(BL_DEFINES)) $(addprefix -I,$(BL_INCLUDE_DIRS)) $(PLAT_BL_COMMON_CPPFLAGS))
+$(eval BL_DEFINES := IMAGE_$(4) $($(4)_DEFINES) $(PLAT_BL_COMMON_DEFINES))
+$(eval BL_INCLUDE_DIRS := $($(4)_INCLUDE_DIRS) $(PLAT_BL_COMMON_INCLUDE_DIRS))
+$(eval BL_CPPFLAGS := $($(4)_CPPFLAGS) $(addprefix -D,$(BL_DEFINES)) $(addprefix -I,$(BL_INCLUDE_DIRS)) $(PLAT_BL_COMMON_CPPFLAGS))
 
 $(1): $(2) $(filter-out %.d,$(MAKEFILE_LIST)) | $$$$(@D)/
 	$$(s)echo "  PP      $$<"
@@ -384,14 +394,15 @@
 #   $(1) = output directory
 #   $(2) = list of source files
 #   $(3) = name of the library
+#   $(4) = uppercase name of the library
 define MAKE_LIB_OBJS
         $(eval C_OBJS := $(filter %.c,$(2)))
         $(eval REMAIN := $(filter-out %.c,$(2)))
-        $(eval $(foreach obj,$(C_OBJS),$(call MAKE_C_LIB,$(1),$(obj),$(3))))
+        $(eval $(foreach obj,$(C_OBJS),$(call MAKE_C_LIB,$(1),$(obj),$(3),$(4))))
 
         $(eval S_OBJS := $(filter %.S,$(REMAIN)))
         $(eval REMAIN := $(filter-out %.S,$(REMAIN)))
-        $(eval $(foreach obj,$(S_OBJS),$(call MAKE_S_LIB,$(1),$(obj),$(3))))
+        $(eval $(foreach obj,$(S_OBJS),$(call MAKE_S_LIB,$(1),$(obj),$(3),$(4))))
 
         $(and $(REMAIN),$(error Unexpected source files present: $(REMAIN)))
 endef
@@ -401,14 +412,15 @@
 #   $(1) = output directory
 #   $(2) = list of source files (both C and assembly)
 #   $(3) = BL stage
+#   $(4) = uppercase BL stage
 define MAKE_OBJS
         $(eval C_OBJS := $(filter %.c,$(2)))
         $(eval REMAIN := $(filter-out %.c,$(2)))
-        $(eval $(foreach obj,$(C_OBJS),$(call MAKE_C,$(1),$(obj),$(3))))
+        $(eval $(foreach obj,$(C_OBJS),$(call MAKE_C,$(1),$(obj),$(3),$(4))))
 
         $(eval S_OBJS := $(filter %.S,$(REMAIN)))
         $(eval REMAIN := $(filter-out %.S,$(REMAIN)))
-        $(eval $(foreach obj,$(S_OBJS),$(call MAKE_S,$(1),$(obj),$(3))))
+        $(eval $(foreach obj,$(S_OBJS),$(call MAKE_S,$(1),$(obj),$(3),$(4))))
 
         $(and $(REMAIN),$(error Unexpected source files present: $(REMAIN)))
 endef
@@ -428,13 +440,14 @@
 # Arguments:
 #   $(1) = Library name
 define MAKE_LIB
+        $(eval BL         := $(call uppercase,$(1)))
         $(eval BUILD_DIR  := ${BUILD_PLAT}/lib$(1))
         $(eval LIB_DIR    := ${BUILD_PLAT}/lib)
         $(eval ROMLIB_DIR    := ${BUILD_PLAT}/romlib)
-        $(eval SOURCES    := $(LIB$(call uppercase,$(1))_SRCS))
+        $(eval SOURCES    := $(LIB$(BL)_SRCS))
         $(eval OBJS       := $(addprefix $(BUILD_DIR)/,$(call SOURCES_TO_OBJS,$(SOURCES))))
 
-$(eval $(call MAKE_LIB_OBJS,$(BUILD_DIR),$(SOURCES),$(1)))
+$(eval $(call MAKE_LIB_OBJS,$(BUILD_DIR),$(SOURCES),$(1),$(BL)))
 
 libraries: ${LIB_DIR}/lib$(1).a
 ifeq ($($(ARCH)-ld-id),arm-link)
@@ -476,8 +489,9 @@
 #   $(3) = FIP prefix (optional) (if FWU_, target is fwu_fip instead of fip)
 #   $(4) = BL encryption flag (optional) (0, 1)
 define MAKE_BL
+        $(eval BL         := $(call uppercase,$(1)))
         $(eval BUILD_DIR  := ${BUILD_PLAT}/$(1))
-        $(eval BL_SOURCES := $($(call uppercase,$(1))_SOURCES))
+        $(eval BL_SOURCES := $($(BL)_SOURCES))
         $(eval SOURCES    := $(sort $(BL_SOURCES) $(BL_COMMON_SOURCES) $(PLAT_BL_COMMON_SOURCES)))
         $(eval OBJS       := $(addprefix $(BUILD_DIR)/,$(call SOURCES_TO_OBJS,$(SOURCES))))
         $(eval MAPFILE    := $(call IMG_MAPFILE,$(1)))
@@ -485,21 +499,21 @@
         $(eval DUMP       := $(call IMG_DUMP,$(1)))
         $(eval BIN        := $(call IMG_BIN,$(1)))
         $(eval ENC_BIN    := $(call IMG_ENC_BIN,$(1)))
-        $(eval BL_LIBS    := $($(call uppercase,$(1))_LIBS))
+        $(eval BL_LIBS    := $($(BL)_LIBS))
 
-        $(eval DEFAULT_LINKER_SCRIPT_SOURCE := $($(call uppercase,$(1))_DEFAULT_LINKER_SCRIPT_SOURCE))
+        $(eval DEFAULT_LINKER_SCRIPT_SOURCE := $($(BL)_DEFAULT_LINKER_SCRIPT_SOURCE))
         $(eval DEFAULT_LINKER_SCRIPT := $(call linker_script_path,$(DEFAULT_LINKER_SCRIPT_SOURCE)))
 
-        $(eval LINKER_SCRIPT_SOURCES := $($(call uppercase,$(1))_LINKER_SCRIPT_SOURCES))
+        $(eval LINKER_SCRIPT_SOURCES := $($(BL)_LINKER_SCRIPT_SOURCES))
         $(eval LINKER_SCRIPTS := $(call linker_script_path,$(LINKER_SCRIPT_SOURCES)))
 
-$(eval $(call MAKE_OBJS,$(BUILD_DIR),$(SOURCES),$(1)))
+$(eval $(call MAKE_OBJS,$(BUILD_DIR),$(SOURCES),$(1),$(BL)))
 
 # Generate targets to preprocess each required linker script
 $(eval $(foreach source,$(DEFAULT_LINKER_SCRIPT_SOURCE) $(LINKER_SCRIPT_SOURCES), \
-        $(call MAKE_LD,$(call linker_script_path,$(source)),$(source),$(1))))
+        $(call MAKE_LD,$(call linker_script_path,$(source)),$(source),$(1),$(BL))))
 
-$(eval BL_LDFLAGS := $($(call uppercase,$(1))_LDFLAGS))
+$(eval BL_LDFLAGS := $($(BL)_LDFLAGS))
 
 ifeq ($(USE_ROMLIB),1)
 $(ELF): romlib.bin | $$$$(@D)/
@@ -554,10 +568,10 @@
 
 ifeq ($(4),1)
 $(call ENCRYPT_FW,$(BIN),$(ENC_BIN))
-$(if $(2),$(call TOOL_ADD_IMG_PAYLOAD,$(1),$(BIN),--$(2),$(ENC_BIN),$(3), \
+$(if $(2),$(call TOOL_ADD_IMG_PAYLOAD,$(BL),$(BIN),--$(2),$(ENC_BIN),$(3), \
 		$(ENC_BIN)))
 else
-$(if $(2),$(call TOOL_ADD_IMG_PAYLOAD,$(1),$(BIN),--$(2),$(BIN),$(3)))
+$(if $(2),$(call TOOL_ADD_IMG_PAYLOAD,$(BL),$(BIN),--$(2),$(BIN),$(3)))
 endif
 
 endef
diff --git a/make_helpers/common.mk b/make_helpers/common.mk
index 75d9f71..848e4e9 100644
--- a/make_helpers/common.mk
+++ b/make_helpers/common.mk
@@ -9,9 +9,9 @@
 
         include $(dir $(common-mk))utilities.mk
 
-        silent = $(call bool,$(findstring s,$(firstword ~$(MAKEFLAGS))))
-        verbose = $(if $(silent),,$(call bool,$(V)))
+        silent := $(call bool,$(findstring s,$(firstword ~$(MAKEFLAGS))))
+        verbose := $(if $(silent),,$(call bool,$(V)))
 
-        s = @$(if $(or $(verbose),$(silent)),: )
-        q = $(if $(verbose),,@)
+        s := @$(if $(or $(verbose),$(silent)),: )
+        q := $(if $(verbose),,@)
 endif
diff --git a/make_helpers/defaults.mk b/make_helpers/defaults.mk
index 584542c..d6c09de 100644
--- a/make_helpers/defaults.mk
+++ b/make_helpers/defaults.mk
@@ -240,6 +240,10 @@
 # region, platform Makefile is free to override this value.
 SEPARATE_BL2_NOLOAD_REGION	:= 0
 
+# Put RW DATA sections (.rwdata) in a separate memory region, which may be
+# discontiguous from the rest of BL31.
+SEPARATE_RWDATA_REGION		:= 0
+
 # Put SIMD context data structures in a separate memory region. Platforms
 # have the choice to put it outside of default BSS region of EL3 firmware.
 SEPARATE_SIMD_SECTION		:= 0
diff --git a/plat/arm/board/arm_fpga/platform.mk b/plat/arm/board/arm_fpga/platform.mk
index 82401db..967bf21 100644
--- a/plat/arm/board/arm_fpga/platform.mk
+++ b/plat/arm/board/arm_fpga/platform.mk
@@ -47,6 +47,7 @@
 ENABLE_TRF_FOR_NS		:= 2
 ENABLE_SME_FOR_NS		:= 2
 ENABLE_SME2_FOR_NS		:= 2
+ENABLE_FEAT_LS64_ACCDATA	:= 2
 
 # Treating this as a memory-constrained port for now
 USE_COHERENT_MEM	:=	0
@@ -127,9 +128,9 @@
 
 BL31_SOURCES		+=	${FDT_WRAPPERS_SOURCES}
 
-$(eval $(call MAKE_S,$(BUILD_PLAT),plat/arm/board/arm_fpga/rom_trampoline.S,bl31))
-$(eval $(call MAKE_S,$(BUILD_PLAT),plat/arm/board/arm_fpga/kernel_trampoline.S,bl31))
-$(eval $(call MAKE_LD,$(BUILD_PLAT)/build_axf.ld,plat/arm/board/arm_fpga/build_axf.ld.S,bl31))
+$(eval $(call MAKE_S,$(BUILD_PLAT),plat/arm/board/arm_fpga/rom_trampoline.S,bl31,BL31))
+$(eval $(call MAKE_S,$(BUILD_PLAT),plat/arm/board/arm_fpga/kernel_trampoline.S,bl31,BL31))
+$(eval $(call MAKE_LD,$(BUILD_PLAT)/build_axf.ld,plat/arm/board/arm_fpga/build_axf.ld.S,bl31,BL31))
 
 ifeq ($($(ARCH)-ld-id),gnu-gcc)
         AXF_LDFLAGS	+=	-Wl,--build-id=none -mno-fix-cortex-a53-843419
diff --git a/plat/arm/board/fvp/platform.mk b/plat/arm/board/fvp/platform.mk
index 6f53a81..0156b31 100644
--- a/plat/arm/board/fvp/platform.mk
+++ b/plat/arm/board/fvp/platform.mk
@@ -80,6 +80,7 @@
 ENABLE_FEAT_S1POE		:= 2
 ENABLE_FEAT_SCTLR2		:= 2
 ENABLE_FEAT_MTE2		:= 2
+ENABLE_FEAT_LS64_ACCDATA	:= 2
 
 # The FVP platform depends on this macro to build with correct GIC driver.
 $(eval $(call add_define,FVP_USE_GIC_DRIVER))