diff --git a/arch/arm/cpu/armv8/fsl-layerscape/cpu.c b/arch/arm/cpu/armv8/fsl-layerscape/cpu.c
index c4bb311..cb3a52c 100644
--- a/arch/arm/cpu/armv8/fsl-layerscape/cpu.c
+++ b/arch/arm/cpu/armv8/fsl-layerscape/cpu.c
@@ -244,6 +244,14 @@
 
 int arch_cpu_init(void)
 {
+	/*
+	 * This function is called before U-Boot relocates itself to speed up
+	 * on system running. It is not necessary to run if performance is not
+	 * critical. Skip if MMU is already enabled by SPL or other means.
+	 */
+	if (get_sctlr() & CR_M)
+		return 0;
+
 	icache_enable();
 	__asm_invalidate_dcache_all();
 	__asm_invalidate_tlb_all();
@@ -530,7 +538,8 @@
 	unsigned long cntfrq = COUNTER_FREQUENCY_REAL;
 
 	/* Update with accurate clock frequency */
-	asm volatile("msr cntfrq_el0, %0" : : "r" (cntfrq) : "memory");
+	if (current_el() == 3)
+		asm volatile("msr cntfrq_el0, %0" : : "r" (cntfrq) : "memory");
 #endif
 
 #ifdef CONFIG_FSL_LSCH3
diff --git a/arch/arm/cpu/armv8/fsl-layerscape/lowlevel.S b/arch/arm/cpu/armv8/fsl-layerscape/lowlevel.S
index f427356..619d9b7 100644
--- a/arch/arm/cpu/armv8/fsl-layerscape/lowlevel.S
+++ b/arch/arm/cpu/armv8/fsl-layerscape/lowlevel.S
@@ -73,6 +73,9 @@
 ENTRY(lowlevel_init)
 	mov	x29, lr			/* Save LR */
 
+	switch_el x1, 1f, 100f, 100f	/* skip if not in EL3 */
+1:
+
 #ifdef CONFIG_FSL_LSCH3
 
 	/* Set Wuo bit for RN-I 20 */
@@ -193,6 +196,7 @@
 #endif
 #endif
 
+100:
 	branch_if_master x0, x1, 2f
 
 #if defined(CONFIG_MP) && defined(CONFIG_ARMV8_MULTIENTRY)
@@ -201,6 +205,8 @@
 #endif
 
 2:
+	switch_el x1, 1f, 100f, 100f	/* skip if not in EL3 */
+1:
 #ifdef CONFIG_FSL_TZPC_BP147
 	/* Set Non Secure access for all devices protected via TZPC */
 	ldr	x1, =TZPCDECPROT_0_SET_BASE /* Decode Protection-0 Set Reg */
@@ -266,8 +272,11 @@
 	isb
 	dsb	sy
 #endif
+100:
 1:
 #ifdef CONFIG_ARCH_LS1046A
+	switch_el x1, 1f, 100f, 100f	/* skip if not in EL3 */
+1:
 	/* Initialize the L2 RAM latency */
 	mrs   x1, S3_1_c11_c0_2
 	mov   x0, #0x1C7
@@ -279,6 +288,7 @@
 	orr   x1,  x1, #0x80
 	msr   S3_1_c11_c0_2, x1
 	isb
+100:
 #endif
 
 #if defined(CONFIG_FSL_LSCH2) && !defined(CONFIG_SPL_BUILD)
@@ -379,11 +389,14 @@
 	/*
 	 * Return status in x0
 	 *    success 0
-	 *    tmeout 1 for setting SFONLY, 2 for FAM, 3 for both
+	 *    timeout 1 for setting SFONLY, 2 for FAM, 3 for both
 	 */
 	mov	x29, lr
 	mov	x8, #0
 
+	switch_el x0, 1f, 100f, 100f	/* skip if not in EL3 */
+
+1:
 	dsb	sy
 	mov	x0, #0x1		/* HNFPSTAT_SFONLY */
 	bl	hnf_set_pstate
@@ -401,6 +414,7 @@
 	bl	hnf_pstate_poll
 	cbz	x0, 1f
 	add	x8, x8, #0x2
+100:
 1:
 	mov	x0, x8
 	mov	lr, x29
diff --git a/arch/arm/cpu/armv8/fsl-layerscape/ppa.c b/arch/arm/cpu/armv8/fsl-layerscape/ppa.c
index 26c47a1..35c612d 100644
--- a/arch/arm/cpu/armv8/fsl-layerscape/ppa.c
+++ b/arch/arm/cpu/armv8/fsl-layerscape/ppa.c
@@ -32,6 +32,7 @@
 
 int ppa_init(void)
 {
+	unsigned int el = current_el();
 	void *ppa_fit_addr;
 	u32 *boot_loc_ptr_l, *boot_loc_ptr_h;
 	int ret;
@@ -45,6 +46,12 @@
 #endif
 #endif
 
+	/* Skip if running at lower exception level */
+	if (el < 3) {
+		debug("Skipping PPA init, running at EL%d\n", el);
+		return 0;
+	}
+
 #ifdef CONFIG_SYS_LS_PPA_FW_IN_XIP
 	ppa_fit_addr = (void *)CONFIG_SYS_LS_PPA_FW_ADDR;
 	debug("%s: PPA image load from XIP\n", __func__);
diff --git a/arch/arm/cpu/armv8/fsl-layerscape/soc.c b/arch/arm/cpu/armv8/fsl-layerscape/soc.c
index 76e3af0..0943e83 100644
--- a/arch/arm/cpu/armv8/fsl-layerscape/soc.c
+++ b/arch/arm/cpu/armv8/fsl-layerscape/soc.c
@@ -288,6 +288,10 @@
 	struct ccsr_cci400 __iomem *cci = (void *)CONFIG_SYS_CCI400_ADDR;
 	struct ccsr_ddr __iomem *ddr = (void *)CONFIG_SYS_FSL_DDR_ADDR;
 
+	/* Skip if running at lower exception level */
+	if (current_el() < 3)
+		return;
+
 	/* disables propagation of barrier transactions to DDRC from CCI400 */
 	out_le32(&cci->ctrl_ord, CCI400_CTRLORD_TERM_BARRIER);
 
@@ -304,6 +308,10 @@
 	struct ccsr_ddr __iomem *ddr = (void *)CONFIG_SYS_FSL_DDR_ADDR;
 	u32 tmp;
 
+	/* Skip if running at lower exception level */
+	if (current_el() < 3)
+		return;
+
 	/* enable propagation of barrier transactions to DDRC from CCI400 */
 	out_le32(&cci->ctrl_ord, CCI400_CTRLORD_EN_BARRIER);
 
@@ -455,8 +463,10 @@
 	 * Enable snoop requests and DVM message requests for
 	 * Slave insterface S4 (A53 core cluster)
 	 */
-	out_le32(&cci->slave[4].snoop_ctrl,
-		 CCI400_DVM_MESSAGE_REQ_EN | CCI400_SNOOP_REQ_EN);
+	if (current_el() == 3) {
+		out_le32(&cci->slave[4].snoop_ctrl,
+			 CCI400_DVM_MESSAGE_REQ_EN | CCI400_SNOOP_REQ_EN);
+	}
 
 	/* Erratum */
 	erratum_a008850_early(); /* part 1 of 2 */
diff --git a/arch/arm/cpu/armv8/sec_firmware.c b/arch/arm/cpu/armv8/sec_firmware.c
index 4afa3ad..fffce71 100644
--- a/arch/arm/cpu/armv8/sec_firmware.c
+++ b/arch/arm/cpu/armv8/sec_firmware.c
@@ -224,7 +224,7 @@
  */
 unsigned int sec_firmware_support_psci_version(void)
 {
-	if (sec_firmware_addr & SEC_FIRMWARE_RUNNING)
+	if (current_el() == SEC_FIRMWARE_TARGET_EL)
 		return _sec_firmware_support_psci_version();
 
 	return PSCI_INVALID_VER;
diff --git a/arch/arm/cpu/armv8/start.S b/arch/arm/cpu/armv8/start.S
index 354468b..5c500be 100644
--- a/arch/arm/cpu/armv8/start.S
+++ b/arch/arm/cpu/armv8/start.S
@@ -91,9 +91,12 @@
 	 * this bit should be set for A53/A57/A72.
 	 */
 #ifdef CONFIG_ARMV8_SET_SMPEN
+	switch_el x1, 3f, 1f, 1f
+3:
 	mrs     x0, S3_1_c15_c2_1               /* cpuectlr_el1 */
 	orr     x0, x0, #0x40
 	msr     S3_1_c15_c2_1, x0
+1:
 #endif
 
 	/* Apply ARM core specific erratas */
diff --git a/board/freescale/common/ns_access.c b/board/freescale/common/ns_access.c
index 81c9211..1c2287d 100644
--- a/board/freescale/common/ns_access.c
+++ b/board/freescale/common/ns_access.c
@@ -39,7 +39,10 @@
 
 void enable_layerscape_ns_access(void)
 {
-	enable_devices_ns_access(ns_dev, ARRAY_SIZE(ns_dev));
+#ifdef CONFIG_ARM64
+	if (current_el() == 3)
+#endif
+		enable_devices_ns_access(ns_dev, ARRAY_SIZE(ns_dev));
 }
 
 void set_pcie_ns_access(int pcie, u16 val)