blob: 769117163e132dfc4ba0550c7a75a447140d31e0 [file] [log] [blame]
Achin Gupta9ac63c52014-01-16 12:08:03 +00001/*
Rohit Mathew3dc3cad2022-11-11 18:45:11 +00002 * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved.
Achin Gupta9ac63c52014-01-16 12:08:03 +00003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Achin Gupta9ac63c52014-01-16 12:08:03 +00005 */
6
Dan Handley2bd4ef22014-04-09 13:14:54 +01007#include <arch.h>
Andrew Thoelke38bde412014-03-18 13:46:55 +00008#include <asm_macros.S>
Jan Dabrosfa015982019-12-02 13:30:03 +01009#include <assert_macros.S>
Dan Handley2bd4ef22014-04-09 13:14:54 +010010#include <context.h>
Manish V Badarkhee07e8082020-07-23 12:43:25 +010011#include <el3_common_macros.S>
Achin Gupta9ac63c52014-01-16 12:08:03 +000012
Max Shvetsovbdf502d2020-02-25 13:56:19 +000013#if CTX_INCLUDE_EL2_REGS
Zelalem Aweke5362beb2022-04-04 17:42:48 -050014 .global el2_sysregs_context_save_common
15 .global el2_sysregs_context_restore_common
Zelalem Aweke5362beb2022-04-04 17:42:48 -050016#if CTX_INCLUDE_MTE_REGS
17 .global el2_sysregs_context_save_mte
18 .global el2_sysregs_context_restore_mte
19#endif /* CTX_INCLUDE_MTE_REGS */
Zelalem Aweke5362beb2022-04-04 17:42:48 -050020#if RAS_EXTENSION
21 .global el2_sysregs_context_save_ras
22 .global el2_sysregs_context_restore_ras
23#endif /* RAS_EXTENSION */
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +000024#endif /* CTX_INCLUDE_EL2_REGS */
Max Shvetsovbdf502d2020-02-25 13:56:19 +000025
Yatharth Kochar6c0566c2015-10-02 17:56:48 +010026 .global el1_sysregs_context_save
27 .global el1_sysregs_context_restore
28#if CTX_INCLUDE_FPREGS
29 .global fpregs_context_save
30 .global fpregs_context_restore
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +000031#endif /* CTX_INCLUDE_FPREGS */
Daniel Boulby95fb1aa2022-01-19 11:20:05 +000032 .global prepare_el3_entry
Alexei Fedorovf41355c2019-09-13 14:11:59 +010033 .global restore_gp_pmcr_pauth_regs
Manish V Badarkhee07e8082020-07-23 12:43:25 +010034 .global save_and_update_ptw_el1_sys_regs
Yatharth Kochar6c0566c2015-10-02 17:56:48 +010035 .global el3_exit
36
Max Shvetsovbdf502d2020-02-25 13:56:19 +000037#if CTX_INCLUDE_EL2_REGS
38
39/* -----------------------------------------------------
Zelalem Aweke5362beb2022-04-04 17:42:48 -050040 * The following functions strictly follow the AArch64
Max Shvetsovcf784f72021-03-31 19:00:38 +010041 * PCS to use x9-x16 (temporary caller-saved registers)
Zelalem Aweke5362beb2022-04-04 17:42:48 -050042 * to save/restore EL2 system register context.
43 * el2_sysregs_context_save/restore_common functions
44 * save and restore registers that are common to all
45 * configurations. The rest of the functions save and
46 * restore EL2 system registers that are present when a
47 * particular feature is enabled. All functions assume
48 * that 'x0' is pointing to a 'el2_sys_regs' structure
49 * where the register context will be saved/restored.
Max Shvetsovc9e2c922020-02-17 16:15:47 +000050 *
51 * The following registers are not added.
52 * AMEVCNTVOFF0<n>_EL2
53 * AMEVCNTVOFF1<n>_EL2
54 * ICH_AP0R<n>_EL2
55 * ICH_AP1R<n>_EL2
56 * ICH_LR<n>_EL2
Max Shvetsovbdf502d2020-02-25 13:56:19 +000057 * -----------------------------------------------------
58 */
Zelalem Aweke5362beb2022-04-04 17:42:48 -050059func el2_sysregs_context_save_common
Max Shvetsovbdf502d2020-02-25 13:56:19 +000060 mrs x9, actlr_el2
Max Shvetsovc9e2c922020-02-17 16:15:47 +000061 mrs x10, afsr0_el2
62 stp x9, x10, [x0, #CTX_ACTLR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000063
Max Shvetsovc9e2c922020-02-17 16:15:47 +000064 mrs x11, afsr1_el2
65 mrs x12, amair_el2
66 stp x11, x12, [x0, #CTX_AFSR1_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000067
Max Shvetsovc9e2c922020-02-17 16:15:47 +000068 mrs x13, cnthctl_el2
Max Shvetsovcf784f72021-03-31 19:00:38 +010069 mrs x14, cntvoff_el2
Max Shvetsovc9e2c922020-02-17 16:15:47 +000070 stp x13, x14, [x0, #CTX_CNTHCTL_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000071
Max Shvetsovcf784f72021-03-31 19:00:38 +010072 mrs x15, cptr_el2
73 str x15, [x0, #CTX_CPTR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000074
Arunachalam Ganapathydca591b2020-05-26 11:32:35 +010075#if CTX_INCLUDE_AARCH32_REGS
Max Shvetsovcf784f72021-03-31 19:00:38 +010076 mrs x16, dbgvcr32_el2
77 str x16, [x0, #CTX_DBGVCR32_EL2]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +000078#endif /* CTX_INCLUDE_AARCH32_REGS */
Max Shvetsovbdf502d2020-02-25 13:56:19 +000079
Max Shvetsovcf784f72021-03-31 19:00:38 +010080 mrs x9, elr_el2
81 mrs x10, esr_el2
82 stp x9, x10, [x0, #CTX_ELR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000083
Max Shvetsovcf784f72021-03-31 19:00:38 +010084 mrs x11, far_el2
85 mrs x12, hacr_el2
86 stp x11, x12, [x0, #CTX_FAR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000087
Max Shvetsovcf784f72021-03-31 19:00:38 +010088 mrs x13, hcr_el2
89 mrs x14, hpfar_el2
90 stp x13, x14, [x0, #CTX_HCR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000091
Max Shvetsovcf784f72021-03-31 19:00:38 +010092 mrs x15, hstr_el2
93 mrs x16, ICC_SRE_EL2
94 stp x15, x16, [x0, #CTX_HSTR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000095
Max Shvetsovcf784f72021-03-31 19:00:38 +010096 mrs x9, ICH_HCR_EL2
97 mrs x10, ICH_VMCR_EL2
98 stp x9, x10, [x0, #CTX_ICH_HCR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +000099
Max Shvetsovcf784f72021-03-31 19:00:38 +0100100 mrs x11, mair_el2
101 mrs x12, mdcr_el2
102 stp x11, x12, [x0, #CTX_MAIR_EL2]
103
Max Shvetsovcf784f72021-03-31 19:00:38 +0100104 mrs x14, sctlr_el2
105 str x14, [x0, #CTX_SCTLR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000106
Max Shvetsovcf784f72021-03-31 19:00:38 +0100107 mrs x15, spsr_el2
108 mrs x16, sp_el2
109 stp x15, x16, [x0, #CTX_SPSR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000110
Max Shvetsovcf784f72021-03-31 19:00:38 +0100111 mrs x9, tcr_el2
112 mrs x10, tpidr_el2
113 stp x9, x10, [x0, #CTX_TCR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000114
Max Shvetsovcf784f72021-03-31 19:00:38 +0100115 mrs x11, ttbr0_el2
116 mrs x12, vbar_el2
117 stp x11, x12, [x0, #CTX_TTBR0_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000118
Max Shvetsovcf784f72021-03-31 19:00:38 +0100119 mrs x13, vmpidr_el2
120 mrs x14, vpidr_el2
121 stp x13, x14, [x0, #CTX_VMPIDR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000122
Max Shvetsovcf784f72021-03-31 19:00:38 +0100123 mrs x15, vtcr_el2
124 mrs x16, vttbr_el2
125 stp x15, x16, [x0, #CTX_VTCR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000126 ret
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500127endfunc el2_sysregs_context_save_common
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000128
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500129func el2_sysregs_context_restore_common
Max Shvetsovc9e2c922020-02-17 16:15:47 +0000130 ldp x9, x10, [x0, #CTX_ACTLR_EL2]
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000131 msr actlr_el2, x9
Max Shvetsovc9e2c922020-02-17 16:15:47 +0000132 msr afsr0_el2, x10
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000133
Max Shvetsovc9e2c922020-02-17 16:15:47 +0000134 ldp x11, x12, [x0, #CTX_AFSR1_EL2]
135 msr afsr1_el2, x11
136 msr amair_el2, x12
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000137
Max Shvetsovc9e2c922020-02-17 16:15:47 +0000138 ldp x13, x14, [x0, #CTX_CNTHCTL_EL2]
139 msr cnthctl_el2, x13
Max Shvetsovcf784f72021-03-31 19:00:38 +0100140 msr cntvoff_el2, x14
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000141
Max Shvetsovcf784f72021-03-31 19:00:38 +0100142 ldr x15, [x0, #CTX_CPTR_EL2]
143 msr cptr_el2, x15
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000144
Arunachalam Ganapathydca591b2020-05-26 11:32:35 +0100145#if CTX_INCLUDE_AARCH32_REGS
Max Shvetsovcf784f72021-03-31 19:00:38 +0100146 ldr x16, [x0, #CTX_DBGVCR32_EL2]
147 msr dbgvcr32_el2, x16
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000148#endif /* CTX_INCLUDE_AARCH32_REGS */
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000149
Max Shvetsovcf784f72021-03-31 19:00:38 +0100150 ldp x9, x10, [x0, #CTX_ELR_EL2]
151 msr elr_el2, x9
152 msr esr_el2, x10
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000153
Max Shvetsovcf784f72021-03-31 19:00:38 +0100154 ldp x11, x12, [x0, #CTX_FAR_EL2]
155 msr far_el2, x11
156 msr hacr_el2, x12
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000157
Max Shvetsovcf784f72021-03-31 19:00:38 +0100158 ldp x13, x14, [x0, #CTX_HCR_EL2]
159 msr hcr_el2, x13
160 msr hpfar_el2, x14
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000161
Max Shvetsovcf784f72021-03-31 19:00:38 +0100162 ldp x15, x16, [x0, #CTX_HSTR_EL2]
163 msr hstr_el2, x15
164 msr ICC_SRE_EL2, x16
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000165
Max Shvetsovcf784f72021-03-31 19:00:38 +0100166 ldp x9, x10, [x0, #CTX_ICH_HCR_EL2]
167 msr ICH_HCR_EL2, x9
168 msr ICH_VMCR_EL2, x10
169
170 ldp x11, x12, [x0, #CTX_MAIR_EL2]
171 msr mair_el2, x11
172 msr mdcr_el2, x12
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000173
Max Shvetsovcf784f72021-03-31 19:00:38 +0100174 ldr x14, [x0, #CTX_SCTLR_EL2]
175 msr sctlr_el2, x14
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000176
Max Shvetsovcf784f72021-03-31 19:00:38 +0100177 ldp x15, x16, [x0, #CTX_SPSR_EL2]
178 msr spsr_el2, x15
179 msr sp_el2, x16
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000180
Max Shvetsovcf784f72021-03-31 19:00:38 +0100181 ldp x9, x10, [x0, #CTX_TCR_EL2]
182 msr tcr_el2, x9
183 msr tpidr_el2, x10
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000184
Max Shvetsovcf784f72021-03-31 19:00:38 +0100185 ldp x11, x12, [x0, #CTX_TTBR0_EL2]
186 msr ttbr0_el2, x11
187 msr vbar_el2, x12
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000188
Max Shvetsovcf784f72021-03-31 19:00:38 +0100189 ldp x13, x14, [x0, #CTX_VMPIDR_EL2]
190 msr vmpidr_el2, x13
191 msr vpidr_el2, x14
Manish V Badarkhed73c1ba2020-07-28 07:12:56 +0100192
Max Shvetsovcf784f72021-03-31 19:00:38 +0100193 ldp x15, x16, [x0, #CTX_VTCR_EL2]
194 msr vtcr_el2, x15
195 msr vttbr_el2, x16
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500196 ret
197endfunc el2_sysregs_context_restore_common
198
Max Shvetsovc9e2c922020-02-17 16:15:47 +0000199#if CTX_INCLUDE_MTE_REGS
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500200func el2_sysregs_context_save_mte
201 mrs x9, TFSR_EL2
202 str x9, [x0, #CTX_TFSR_EL2]
203 ret
204endfunc el2_sysregs_context_save_mte
205
206func el2_sysregs_context_restore_mte
Manish V Badarkhed73c1ba2020-07-28 07:12:56 +0100207 ldr x9, [x0, #CTX_TFSR_EL2]
208 msr TFSR_EL2, x9
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500209 ret
210endfunc el2_sysregs_context_restore_mte
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000211#endif /* CTX_INCLUDE_MTE_REGS */
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000212
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000213#if RAS_EXTENSION
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500214func el2_sysregs_context_save_ras
215 /*
216 * VDISR_EL2 and VSESR_EL2 registers are saved only when
217 * FEAT_RAS is supported.
218 */
219 mrs x11, vdisr_el2
220 mrs x12, vsesr_el2
221 stp x11, x12, [x0, #CTX_VDISR_EL2]
222 ret
223endfunc el2_sysregs_context_save_ras
224
225func el2_sysregs_context_restore_ras
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000226 /*
227 * VDISR_EL2 and VSESR_EL2 registers are restored only when FEAT_RAS
228 * is supported.
229 */
230 ldp x11, x12, [x0, #CTX_VDISR_EL2]
231 msr vdisr_el2, x11
232 msr vsesr_el2, x12
Zelalem Aweke5362beb2022-04-04 17:42:48 -0500233 ret
234endfunc el2_sysregs_context_restore_ras
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000235#endif /* RAS_EXTENSION */
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000236
Max Shvetsovbdf502d2020-02-25 13:56:19 +0000237#endif /* CTX_INCLUDE_EL2_REGS */
238
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100239/* ------------------------------------------------------------------
240 * The following function strictly follows the AArch64 PCS to use
241 * x9-x17 (temporary caller-saved registers) to save EL1 system
242 * register context. It assumes that 'x0' is pointing to a
243 * 'el1_sys_regs' structure where the register context will be saved.
244 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000245 */
Andrew Thoelke38bde412014-03-18 13:46:55 +0000246func el1_sysregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000247
248 mrs x9, spsr_el1
249 mrs x10, elr_el1
250 stp x9, x10, [x0, #CTX_SPSR_EL1]
251
Manish V Badarkhee07e8082020-07-23 12:43:25 +0100252#if !ERRATA_SPECULATIVE_AT
Achin Gupta9ac63c52014-01-16 12:08:03 +0000253 mrs x15, sctlr_el1
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100254 mrs x16, tcr_el1
Achin Gupta9ac63c52014-01-16 12:08:03 +0000255 stp x15, x16, [x0, #CTX_SCTLR_EL1]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000256#endif /* ERRATA_SPECULATIVE_AT */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000257
258 mrs x17, cpacr_el1
259 mrs x9, csselr_el1
260 stp x17, x9, [x0, #CTX_CPACR_EL1]
261
262 mrs x10, sp_el1
263 mrs x11, esr_el1
264 stp x10, x11, [x0, #CTX_SP_EL1]
265
266 mrs x12, ttbr0_el1
267 mrs x13, ttbr1_el1
268 stp x12, x13, [x0, #CTX_TTBR0_EL1]
269
270 mrs x14, mair_el1
271 mrs x15, amair_el1
272 stp x14, x15, [x0, #CTX_MAIR_EL1]
273
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100274 mrs x16, actlr_el1
Achin Gupta9ac63c52014-01-16 12:08:03 +0000275 mrs x17, tpidr_el1
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100276 stp x16, x17, [x0, #CTX_ACTLR_EL1]
Achin Gupta9ac63c52014-01-16 12:08:03 +0000277
278 mrs x9, tpidr_el0
279 mrs x10, tpidrro_el0
280 stp x9, x10, [x0, #CTX_TPIDR_EL0]
281
Achin Gupta9ac63c52014-01-16 12:08:03 +0000282 mrs x13, par_el1
283 mrs x14, far_el1
284 stp x13, x14, [x0, #CTX_PAR_EL1]
285
286 mrs x15, afsr0_el1
287 mrs x16, afsr1_el1
288 stp x15, x16, [x0, #CTX_AFSR0_EL1]
289
290 mrs x17, contextidr_el1
291 mrs x9, vbar_el1
292 stp x17, x9, [x0, #CTX_CONTEXTIDR_EL1]
293
Soby Mathewd75d2ba2016-05-17 14:01:32 +0100294 /* Save AArch32 system registers if the build has instructed so */
295#if CTX_INCLUDE_AARCH32_REGS
296 mrs x11, spsr_abt
297 mrs x12, spsr_und
298 stp x11, x12, [x0, #CTX_SPSR_ABT]
299
300 mrs x13, spsr_irq
301 mrs x14, spsr_fiq
302 stp x13, x14, [x0, #CTX_SPSR_IRQ]
303
304 mrs x15, dacr32_el2
305 mrs x16, ifsr32_el2
306 stp x15, x16, [x0, #CTX_DACR32_EL2]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000307#endif /* CTX_INCLUDE_AARCH32_REGS */
Soby Mathewd75d2ba2016-05-17 14:01:32 +0100308
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100309 /* Save NS timer registers if the build has instructed so */
310#if NS_TIMER_SWITCH
Achin Gupta9ac63c52014-01-16 12:08:03 +0000311 mrs x10, cntp_ctl_el0
312 mrs x11, cntp_cval_el0
313 stp x10, x11, [x0, #CTX_CNTP_CTL_EL0]
314
315 mrs x12, cntv_ctl_el0
316 mrs x13, cntv_cval_el0
317 stp x12, x13, [x0, #CTX_CNTV_CTL_EL0]
318
319 mrs x14, cntkctl_el1
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100320 str x14, [x0, #CTX_CNTKCTL_EL1]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000321#endif /* NS_TIMER_SWITCH */
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100322
Justin Chadwell1c7c13a2019-07-18 14:25:33 +0100323 /* Save MTE system registers if the build has instructed so */
324#if CTX_INCLUDE_MTE_REGS
325 mrs x15, TFSRE0_EL1
326 mrs x16, TFSR_EL1
327 stp x15, x16, [x0, #CTX_TFSRE0_EL1]
328
329 mrs x9, RGSR_EL1
330 mrs x10, GCR_EL1
331 stp x9, x10, [x0, #CTX_RGSR_EL1]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000332#endif /* CTX_INCLUDE_MTE_REGS */
Justin Chadwell1c7c13a2019-07-18 14:25:33 +0100333
Achin Gupta9ac63c52014-01-16 12:08:03 +0000334 ret
Kévin Petita877c252015-03-24 14:03:57 +0000335endfunc el1_sysregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000336
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100337/* ------------------------------------------------------------------
338 * The following function strictly follows the AArch64 PCS to use
339 * x9-x17 (temporary caller-saved registers) to restore EL1 system
340 * register context. It assumes that 'x0' is pointing to a
341 * 'el1_sys_regs' structure from where the register context will be
342 * restored
343 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000344 */
Andrew Thoelke38bde412014-03-18 13:46:55 +0000345func el1_sysregs_context_restore
Achin Gupta9ac63c52014-01-16 12:08:03 +0000346
347 ldp x9, x10, [x0, #CTX_SPSR_EL1]
348 msr spsr_el1, x9
349 msr elr_el1, x10
350
Manish V Badarkhee07e8082020-07-23 12:43:25 +0100351#if !ERRATA_SPECULATIVE_AT
Manish V Badarkhed73c1ba2020-07-28 07:12:56 +0100352 ldp x15, x16, [x0, #CTX_SCTLR_EL1]
353 msr sctlr_el1, x15
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100354 msr tcr_el1, x16
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000355#endif /* ERRATA_SPECULATIVE_AT */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000356
357 ldp x17, x9, [x0, #CTX_CPACR_EL1]
358 msr cpacr_el1, x17
359 msr csselr_el1, x9
360
361 ldp x10, x11, [x0, #CTX_SP_EL1]
362 msr sp_el1, x10
363 msr esr_el1, x11
364
365 ldp x12, x13, [x0, #CTX_TTBR0_EL1]
366 msr ttbr0_el1, x12
367 msr ttbr1_el1, x13
368
369 ldp x14, x15, [x0, #CTX_MAIR_EL1]
370 msr mair_el1, x14
371 msr amair_el1, x15
372
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100373 ldp x16, x17, [x0, #CTX_ACTLR_EL1]
374 msr actlr_el1, x16
Manish V Badarkhed73c1ba2020-07-28 07:12:56 +0100375 msr tpidr_el1, x17
Achin Gupta9ac63c52014-01-16 12:08:03 +0000376
377 ldp x9, x10, [x0, #CTX_TPIDR_EL0]
378 msr tpidr_el0, x9
379 msr tpidrro_el0, x10
380
Achin Gupta9ac63c52014-01-16 12:08:03 +0000381 ldp x13, x14, [x0, #CTX_PAR_EL1]
382 msr par_el1, x13
383 msr far_el1, x14
384
385 ldp x15, x16, [x0, #CTX_AFSR0_EL1]
386 msr afsr0_el1, x15
387 msr afsr1_el1, x16
388
389 ldp x17, x9, [x0, #CTX_CONTEXTIDR_EL1]
390 msr contextidr_el1, x17
391 msr vbar_el1, x9
392
Soby Mathewd75d2ba2016-05-17 14:01:32 +0100393 /* Restore AArch32 system registers if the build has instructed so */
394#if CTX_INCLUDE_AARCH32_REGS
395 ldp x11, x12, [x0, #CTX_SPSR_ABT]
396 msr spsr_abt, x11
397 msr spsr_und, x12
398
399 ldp x13, x14, [x0, #CTX_SPSR_IRQ]
400 msr spsr_irq, x13
401 msr spsr_fiq, x14
402
403 ldp x15, x16, [x0, #CTX_DACR32_EL2]
404 msr dacr32_el2, x15
405 msr ifsr32_el2, x16
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000406#endif /* CTX_INCLUDE_AARCH32_REGS */
407
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100408 /* Restore NS timer registers if the build has instructed so */
409#if NS_TIMER_SWITCH
Achin Gupta9ac63c52014-01-16 12:08:03 +0000410 ldp x10, x11, [x0, #CTX_CNTP_CTL_EL0]
411 msr cntp_ctl_el0, x10
412 msr cntp_cval_el0, x11
413
414 ldp x12, x13, [x0, #CTX_CNTV_CTL_EL0]
415 msr cntv_ctl_el0, x12
416 msr cntv_cval_el0, x13
417
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100418 ldr x14, [x0, #CTX_CNTKCTL_EL1]
Achin Gupta9ac63c52014-01-16 12:08:03 +0000419 msr cntkctl_el1, x14
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000420#endif /* NS_TIMER_SWITCH */
421
Justin Chadwell1c7c13a2019-07-18 14:25:33 +0100422 /* Restore MTE system registers if the build has instructed so */
423#if CTX_INCLUDE_MTE_REGS
424 ldp x11, x12, [x0, #CTX_TFSRE0_EL1]
425 msr TFSRE0_EL1, x11
426 msr TFSR_EL1, x12
427
428 ldp x13, x14, [x0, #CTX_RGSR_EL1]
429 msr RGSR_EL1, x13
430 msr GCR_EL1, x14
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000431#endif /* CTX_INCLUDE_MTE_REGS */
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100432
Achin Gupta9ac63c52014-01-16 12:08:03 +0000433 /* No explict ISB required here as ERET covers it */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000434 ret
Kévin Petita877c252015-03-24 14:03:57 +0000435endfunc el1_sysregs_context_restore
Achin Gupta9ac63c52014-01-16 12:08:03 +0000436
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100437/* ------------------------------------------------------------------
438 * The following function follows the aapcs_64 strictly to use
439 * x9-x17 (temporary caller-saved registers according to AArch64 PCS)
440 * to save floating point register context. It assumes that 'x0' is
441 * pointing to a 'fp_regs' structure where the register context will
Achin Gupta9ac63c52014-01-16 12:08:03 +0000442 * be saved.
443 *
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100444 * Access to VFP registers will trap if CPTR_EL3.TFP is set.
445 * However currently we don't use VFP registers nor set traps in
446 * Trusted Firmware, and assume it's cleared.
Achin Gupta9ac63c52014-01-16 12:08:03 +0000447 *
448 * TODO: Revisit when VFP is used in secure world
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100449 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000450 */
Juan Castillo258e94f2014-06-25 17:26:36 +0100451#if CTX_INCLUDE_FPREGS
Andrew Thoelke38bde412014-03-18 13:46:55 +0000452func fpregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000453 stp q0, q1, [x0, #CTX_FP_Q0]
454 stp q2, q3, [x0, #CTX_FP_Q2]
455 stp q4, q5, [x0, #CTX_FP_Q4]
456 stp q6, q7, [x0, #CTX_FP_Q6]
457 stp q8, q9, [x0, #CTX_FP_Q8]
458 stp q10, q11, [x0, #CTX_FP_Q10]
459 stp q12, q13, [x0, #CTX_FP_Q12]
460 stp q14, q15, [x0, #CTX_FP_Q14]
461 stp q16, q17, [x0, #CTX_FP_Q16]
462 stp q18, q19, [x0, #CTX_FP_Q18]
463 stp q20, q21, [x0, #CTX_FP_Q20]
464 stp q22, q23, [x0, #CTX_FP_Q22]
465 stp q24, q25, [x0, #CTX_FP_Q24]
466 stp q26, q27, [x0, #CTX_FP_Q26]
467 stp q28, q29, [x0, #CTX_FP_Q28]
468 stp q30, q31, [x0, #CTX_FP_Q30]
469
470 mrs x9, fpsr
471 str x9, [x0, #CTX_FP_FPSR]
472
473 mrs x10, fpcr
474 str x10, [x0, #CTX_FP_FPCR]
475
David Cunadod1a1fd42017-10-20 11:30:57 +0100476#if CTX_INCLUDE_AARCH32_REGS
477 mrs x11, fpexc32_el2
478 str x11, [x0, #CTX_FP_FPEXC32_EL2]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000479#endif /* CTX_INCLUDE_AARCH32_REGS */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000480 ret
Kévin Petita877c252015-03-24 14:03:57 +0000481endfunc fpregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000482
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100483/* ------------------------------------------------------------------
484 * The following function follows the aapcs_64 strictly to use x9-x17
485 * (temporary caller-saved registers according to AArch64 PCS) to
486 * restore floating point register context. It assumes that 'x0' is
487 * pointing to a 'fp_regs' structure from where the register context
Achin Gupta9ac63c52014-01-16 12:08:03 +0000488 * will be restored.
489 *
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100490 * Access to VFP registers will trap if CPTR_EL3.TFP is set.
491 * However currently we don't use VFP registers nor set traps in
492 * Trusted Firmware, and assume it's cleared.
Achin Gupta9ac63c52014-01-16 12:08:03 +0000493 *
494 * TODO: Revisit when VFP is used in secure world
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100495 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000496 */
Andrew Thoelke38bde412014-03-18 13:46:55 +0000497func fpregs_context_restore
Achin Gupta9ac63c52014-01-16 12:08:03 +0000498 ldp q0, q1, [x0, #CTX_FP_Q0]
499 ldp q2, q3, [x0, #CTX_FP_Q2]
500 ldp q4, q5, [x0, #CTX_FP_Q4]
501 ldp q6, q7, [x0, #CTX_FP_Q6]
502 ldp q8, q9, [x0, #CTX_FP_Q8]
503 ldp q10, q11, [x0, #CTX_FP_Q10]
504 ldp q12, q13, [x0, #CTX_FP_Q12]
505 ldp q14, q15, [x0, #CTX_FP_Q14]
506 ldp q16, q17, [x0, #CTX_FP_Q16]
507 ldp q18, q19, [x0, #CTX_FP_Q18]
508 ldp q20, q21, [x0, #CTX_FP_Q20]
509 ldp q22, q23, [x0, #CTX_FP_Q22]
510 ldp q24, q25, [x0, #CTX_FP_Q24]
511 ldp q26, q27, [x0, #CTX_FP_Q26]
512 ldp q28, q29, [x0, #CTX_FP_Q28]
513 ldp q30, q31, [x0, #CTX_FP_Q30]
514
515 ldr x9, [x0, #CTX_FP_FPSR]
516 msr fpsr, x9
517
Soby Mathewe77e1162015-12-03 09:42:50 +0000518 ldr x10, [x0, #CTX_FP_FPCR]
Achin Gupta9ac63c52014-01-16 12:08:03 +0000519 msr fpcr, x10
520
David Cunadod1a1fd42017-10-20 11:30:57 +0100521#if CTX_INCLUDE_AARCH32_REGS
522 ldr x11, [x0, #CTX_FP_FPEXC32_EL2]
523 msr fpexc32_el2, x11
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000524#endif /* CTX_INCLUDE_AARCH32_REGS */
525
Achin Gupta9ac63c52014-01-16 12:08:03 +0000526 /*
527 * No explict ISB required here as ERET to
Sandrine Bailleuxf4119ec2015-12-17 13:58:58 +0000528 * switch to secure EL1 or non-secure world
Achin Gupta9ac63c52014-01-16 12:08:03 +0000529 * covers it
530 */
531
532 ret
Kévin Petita877c252015-03-24 14:03:57 +0000533endfunc fpregs_context_restore
Juan Castillo258e94f2014-06-25 17:26:36 +0100534#endif /* CTX_INCLUDE_FPREGS */
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100535
Daniel Boulby928747f2021-05-25 18:09:34 +0100536 /*
Manish Pandey62d532a2022-11-17 15:47:05 +0000537 * Set SCR_EL3.EA bit to enable SErrors at EL3
538 */
539 .macro enable_serror_at_el3
540 mrs x8, scr_el3
541 orr x8, x8, #SCR_EA_BIT
542 msr scr_el3, x8
543 .endm
544
545 /*
Daniel Boulby928747f2021-05-25 18:09:34 +0100546 * Set the PSTATE bits not set when the exception was taken as
547 * described in the AArch64.TakeException() pseudocode function
548 * in ARM DDI 0487F.c page J1-7635 to a default value.
549 */
550 .macro set_unset_pstate_bits
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000551 /*
552 * If Data Independent Timing (DIT) functionality is implemented,
553 * always enable DIT in EL3
554 */
Daniel Boulby928747f2021-05-25 18:09:34 +0100555#if ENABLE_FEAT_DIT
Andre Przywara1f55c412023-01-26 16:47:52 +0000556#if ENABLE_FEAT_DIT == 2
557 mrs x8, id_aa64pfr0_el1
558 and x8, x8, #(ID_AA64PFR0_DIT_MASK << ID_AA64PFR0_DIT_SHIFT)
559 cbz x8, 1f
560#endif
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000561 mov x8, #DIT_BIT
562 msr DIT, x8
Andre Przywara1f55c412023-01-26 16:47:52 +00005631:
Daniel Boulby928747f2021-05-25 18:09:34 +0100564#endif /* ENABLE_FEAT_DIT */
565 .endm /* set_unset_pstate_bits */
566
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100567/* ------------------------------------------------------------------
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000568 * The following macro is used to save and restore all the general
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100569 * purpose and ARMv8.3-PAuth (if enabled) registers.
Jayanth Dodderi Chidanand4ec78ad2022-09-19 23:32:08 +0100570 * It also checks if the Secure Cycle Counter (PMCCNTR_EL0)
571 * is disabled in EL3/Secure (ARMv8.5-PMU), wherein PMCCNTR_EL0
572 * needs not to be saved/restored during world switch.
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100573 *
574 * Ideally we would only save and restore the callee saved registers
575 * when a world switch occurs but that type of implementation is more
576 * complex. So currently we will always save and restore these
577 * registers on entry and exit of EL3.
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100578 * clobbers: x18
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100579 * ------------------------------------------------------------------
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100580 */
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000581 .macro save_gp_pmcr_pauth_regs
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100582 stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
583 stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
584 stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
585 stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
586 stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
587 stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
588 stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
589 stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
590 stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
591 stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
592 stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
593 stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
594 stp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
595 stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
596 stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
597 mrs x18, sp_el0
598 str x18, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0]
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100599
600 /* ----------------------------------------------------------
Jayanth Dodderi Chidanand4ec78ad2022-09-19 23:32:08 +0100601 * Check if earlier initialization of MDCR_EL3.SCCD/MCCD to 1
602 * has failed.
603 *
604 * MDCR_EL3:
605 * MCCD bit set, Prohibits the Cycle Counter PMCCNTR_EL0 from
606 * counting at EL3.
607 * SCCD bit set, Secure Cycle Counter Disable. Prohibits PMCCNTR_EL0
608 * from counting in Secure state.
609 * If these bits are not set, meaning that FEAT_PMUv3p5/7 is
610 * not implemented and PMCR_EL0 should be saved in non-secure
611 * context.
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100612 * ----------------------------------------------------------
613 */
Alexei Fedorov307f34b2021-05-14 11:21:56 +0100614 mov_imm x10, (MDCR_SCCD_BIT | MDCR_MCCD_BIT)
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100615 mrs x9, mdcr_el3
Alexei Fedorov307f34b2021-05-14 11:21:56 +0100616 tst x9, x10
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100617 bne 1f
618
Jayanth Dodderi Chidanand4ec78ad2022-09-19 23:32:08 +0100619 /* ----------------------------------------------------------
620 * If control reaches here, it ensures the Secure Cycle
621 * Counter (PMCCNTR_EL0) is not prohibited from counting at
622 * EL3 and in secure states.
623 * Henceforth, PMCR_EL0 to be saved before world switch.
624 * ----------------------------------------------------------
625 */
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100626 mrs x9, pmcr_el0
627
628 /* Check caller's security state */
629 mrs x10, scr_el3
630 tst x10, #SCR_NS_BIT
631 beq 2f
632
633 /* Save PMCR_EL0 if called from Non-secure state */
634 str x9, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0]
635
636 /* Disable cycle counter when event counting is prohibited */
6372: orr x9, x9, #PMCR_EL0_DP_BIT
638 msr pmcr_el0, x9
639 isb
6401:
641#if CTX_INCLUDE_PAUTH_REGS
642 /* ----------------------------------------------------------
643 * Save the ARMv8.3-PAuth keys as they are not banked
644 * by exception level
645 * ----------------------------------------------------------
646 */
647 add x19, sp, #CTX_PAUTH_REGS_OFFSET
648
649 mrs x20, APIAKeyLo_EL1 /* x21:x20 = APIAKey */
650 mrs x21, APIAKeyHi_EL1
651 mrs x22, APIBKeyLo_EL1 /* x23:x22 = APIBKey */
652 mrs x23, APIBKeyHi_EL1
653 mrs x24, APDAKeyLo_EL1 /* x25:x24 = APDAKey */
654 mrs x25, APDAKeyHi_EL1
655 mrs x26, APDBKeyLo_EL1 /* x27:x26 = APDBKey */
656 mrs x27, APDBKeyHi_EL1
657 mrs x28, APGAKeyLo_EL1 /* x29:x28 = APGAKey */
658 mrs x29, APGAKeyHi_EL1
659
660 stp x20, x21, [x19, #CTX_PACIAKEY_LO]
661 stp x22, x23, [x19, #CTX_PACIBKEY_LO]
662 stp x24, x25, [x19, #CTX_PACDAKEY_LO]
663 stp x26, x27, [x19, #CTX_PACDBKEY_LO]
664 stp x28, x29, [x19, #CTX_PACGAKEY_LO]
665#endif /* CTX_INCLUDE_PAUTH_REGS */
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000666 .endm /* save_gp_pmcr_pauth_regs */
667
668/* -----------------------------------------------------------------
Daniel Boulby928747f2021-05-25 18:09:34 +0100669 * This function saves the context and sets the PSTATE to a known
670 * state, preparing entry to el3.
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000671 * Save all the general purpose and ARMv8.3-PAuth (if enabled)
672 * registers.
Daniel Boulby928747f2021-05-25 18:09:34 +0100673 * Then set any of the PSTATE bits that are not set by hardware
674 * according to the Aarch64.TakeException pseudocode in the Arm
675 * Architecture Reference Manual to a default value for EL3.
676 * clobbers: x17
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000677 * -----------------------------------------------------------------
678 */
679func prepare_el3_entry
680 save_gp_pmcr_pauth_regs
Manish Pandey62d532a2022-11-17 15:47:05 +0000681 enable_serror_at_el3
Daniel Boulby928747f2021-05-25 18:09:34 +0100682 /*
683 * Set the PSTATE bits not described in the Aarch64.TakeException
684 * pseudocode to their default values.
685 */
686 set_unset_pstate_bits
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100687 ret
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000688endfunc prepare_el3_entry
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100689
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100690/* ------------------------------------------------------------------
691 * This function restores ARMv8.3-PAuth (if enabled) and all general
692 * purpose registers except x30 from the CPU context.
693 * x30 register must be explicitly restored by the caller.
694 * ------------------------------------------------------------------
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000695 */
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100696func restore_gp_pmcr_pauth_regs
697#if CTX_INCLUDE_PAUTH_REGS
698 /* Restore the ARMv8.3 PAuth keys */
699 add x10, sp, #CTX_PAUTH_REGS_OFFSET
700
701 ldp x0, x1, [x10, #CTX_PACIAKEY_LO] /* x1:x0 = APIAKey */
702 ldp x2, x3, [x10, #CTX_PACIBKEY_LO] /* x3:x2 = APIBKey */
703 ldp x4, x5, [x10, #CTX_PACDAKEY_LO] /* x5:x4 = APDAKey */
704 ldp x6, x7, [x10, #CTX_PACDBKEY_LO] /* x7:x6 = APDBKey */
705 ldp x8, x9, [x10, #CTX_PACGAKEY_LO] /* x9:x8 = APGAKey */
706
707 msr APIAKeyLo_EL1, x0
708 msr APIAKeyHi_EL1, x1
709 msr APIBKeyLo_EL1, x2
710 msr APIBKeyHi_EL1, x3
711 msr APDAKeyLo_EL1, x4
712 msr APDAKeyHi_EL1, x5
713 msr APDBKeyLo_EL1, x6
714 msr APDBKeyHi_EL1, x7
715 msr APGAKeyLo_EL1, x8
716 msr APGAKeyHi_EL1, x9
717#endif /* CTX_INCLUDE_PAUTH_REGS */
718
719 /* ----------------------------------------------------------
720 * Restore PMCR_EL0 when returning to Non-secure state if
721 * Secure Cycle Counter is not disabled in MDCR_EL3 when
722 * ARMv8.5-PMU is implemented.
723 * ----------------------------------------------------------
724 */
725 mrs x0, scr_el3
726 tst x0, #SCR_NS_BIT
727 beq 2f
728
729 /* ----------------------------------------------------------
730 * Back to Non-secure state.
Alexei Fedorov307f34b2021-05-14 11:21:56 +0100731 * Check if earlier initialization MDCR_EL3.SCCD/MCCD to 1
732 * failed, meaning that FEAT_PMUv3p5/7 is not implemented and
733 * PMCR_EL0 should be restored from non-secure context.
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100734 * ----------------------------------------------------------
735 */
Alexei Fedorov307f34b2021-05-14 11:21:56 +0100736 mov_imm x1, (MDCR_SCCD_BIT | MDCR_MCCD_BIT)
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100737 mrs x0, mdcr_el3
Alexei Fedorov307f34b2021-05-14 11:21:56 +0100738 tst x0, x1
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100739 bne 2f
740 ldr x0, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0]
741 msr pmcr_el0, x0
7422:
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100743 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
744 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100745 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
746 ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
747 ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
748 ldp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
749 ldp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
750 ldp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000751 ldp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100752 ldp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
753 ldp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
754 ldp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
755 ldp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
756 ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000757 ldr x28, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0]
758 msr sp_el0, x28
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100759 ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000760 ret
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100761endfunc restore_gp_pmcr_pauth_regs
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000762
Manish V Badarkhee07e8082020-07-23 12:43:25 +0100763/*
764 * In case of ERRATA_SPECULATIVE_AT, save SCTLR_EL1 and TCR_EL1
765 * registers and update EL1 registers to disable stage1 and stage2
766 * page table walk
767 */
768func save_and_update_ptw_el1_sys_regs
769 /* ----------------------------------------------------------
770 * Save only sctlr_el1 and tcr_el1 registers
771 * ----------------------------------------------------------
772 */
773 mrs x29, sctlr_el1
774 str x29, [sp, #(CTX_EL1_SYSREGS_OFFSET + CTX_SCTLR_EL1)]
775 mrs x29, tcr_el1
776 str x29, [sp, #(CTX_EL1_SYSREGS_OFFSET + CTX_TCR_EL1)]
777
778 /* ------------------------------------------------------------
779 * Must follow below order in order to disable page table
780 * walk for lower ELs (EL1 and EL0). First step ensures that
781 * page table walk is disabled for stage1 and second step
782 * ensures that page table walker should use TCR_EL1.EPDx
783 * bits to perform address translation. ISB ensures that CPU
784 * does these 2 steps in order.
785 *
786 * 1. Update TCR_EL1.EPDx bits to disable page table walk by
787 * stage1.
788 * 2. Enable MMU bit to avoid identity mapping via stage2
789 * and force TCR_EL1.EPDx to be used by the page table
790 * walker.
791 * ------------------------------------------------------------
792 */
793 orr x29, x29, #(TCR_EPD0_BIT)
794 orr x29, x29, #(TCR_EPD1_BIT)
795 msr tcr_el1, x29
796 isb
797 mrs x29, sctlr_el1
798 orr x29, x29, #SCTLR_M_BIT
799 msr sctlr_el1, x29
800 isb
801
802 ret
803endfunc save_and_update_ptw_el1_sys_regs
804
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100805/* ------------------------------------------------------------------
806 * This routine assumes that the SP_EL3 is pointing to a valid
807 * context structure from where the gp regs and other special
808 * registers can be retrieved.
809 * ------------------------------------------------------------------
Antonio Nino Diaz13adfb12019-01-30 20:41:31 +0000810 */
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100811func el3_exit
Jan Dabrosfa015982019-12-02 13:30:03 +0100812#if ENABLE_ASSERTIONS
813 /* el3_exit assumes SP_EL0 on entry */
814 mrs x17, spsel
815 cmp x17, #MODE_SP_EL0
816 ASM_ASSERT(eq)
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000817#endif /* ENABLE_ASSERTIONS */
Jan Dabrosfa015982019-12-02 13:30:03 +0100818
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100819 /* ----------------------------------------------------------
820 * Save the current SP_EL0 i.e. the EL3 runtime stack which
821 * will be used for handling the next SMC.
822 * Then switch to SP_EL3.
823 * ----------------------------------------------------------
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100824 */
825 mov x17, sp
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100826 msr spsel, #MODE_SP_ELX
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100827 str x17, [sp, #CTX_EL3STATE_OFFSET + CTX_RUNTIME_SP]
828
Max Shvetsovc4502772021-03-22 11:59:37 +0000829#if IMAGE_BL31
830 /* ----------------------------------------------------------
Arunachalam Ganapathycac7d162021-07-08 09:35:57 +0100831 * Restore CPTR_EL3.
Max Shvetsovc4502772021-03-22 11:59:37 +0000832 * ZCR is only restored if SVE is supported and enabled.
833 * Synchronization is required before zcr_el3 is addressed.
834 * ----------------------------------------------------------
835 */
Max Shvetsovc4502772021-03-22 11:59:37 +0000836 ldp x19, x20, [sp, #CTX_EL3STATE_OFFSET + CTX_CPTR_EL3]
837 msr cptr_el3, x19
838
839 ands x19, x19, #CPTR_EZ_BIT
840 beq sve_not_enabled
841
842 isb
843 msr S3_6_C1_C2_0, x20 /* zcr_el3 */
844sve_not_enabled:
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000845#endif /* IMAGE_BL31 */
Max Shvetsovc4502772021-03-22 11:59:37 +0000846
Dimitris Papastamosba51d9e2018-05-16 11:36:14 +0100847#if IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100848 /* ----------------------------------------------------------
849 * Restore mitigation state as it was on entry to EL3
850 * ----------------------------------------------------------
851 */
Dimitris Papastamosba51d9e2018-05-16 11:36:14 +0100852 ldr x17, [sp, #CTX_CVE_2018_3639_OFFSET + CTX_CVE_2018_3639_DISABLE]
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100853 cbz x17, 1f
Dimitris Papastamosba51d9e2018-05-16 11:36:14 +0100854 blr x17
Antonio Nino Diaz13adfb12019-01-30 20:41:31 +00008551:
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000856#endif /* IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 */
857
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100858#if IMAGE_BL31 && RAS_EXTENSION
859 /* ----------------------------------------------------------
860 * Issue Error Synchronization Barrier to synchronize SErrors
861 * before exiting EL3. We're running with EAs unmasked, so
862 * any synchronized errors would be taken immediately;
863 * therefore no need to inspect DISR_EL1 register.
864 * ----------------------------------------------------------
865 */
866 esb
Madhukar Pappireddyfba25722020-07-24 03:27:12 -0500867#else
868 dsb sy
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000869#endif /* IMAGE_BL31 && RAS_EXTENSION */
870
Manish Pandey53bc59a2022-11-17 14:43:15 +0000871 /* ----------------------------------------------------------
872 * Restore SPSR_EL3, ELR_EL3 and SCR_EL3 prior to ERET
873 * ----------------------------------------------------------
874 */
875 ldr x18, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3]
876 ldp x16, x17, [sp, #CTX_EL3STATE_OFFSET + CTX_SPSR_EL3]
877 msr scr_el3, x18
878 msr spsr_el3, x16
879 msr elr_el3, x17
880
881 restore_ptw_el1_sys_regs
882
883 /* ----------------------------------------------------------
884 * Restore general purpose (including x30), PMCR_EL0 and
885 * ARMv8.3-PAuth registers.
886 * Exit EL3 via ERET to a lower exception level.
887 * ----------------------------------------------------------
888 */
889 bl restore_gp_pmcr_pauth_regs
890 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
891
Madhukar Pappireddyfba25722020-07-24 03:27:12 -0500892#ifdef IMAGE_BL31
893 str xzr, [sp, #CTX_EL3STATE_OFFSET + CTX_IS_IN_EL3]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000894#endif /* IMAGE_BL31 */
895
Anthony Steinhauser0f7e6012020-01-07 15:44:06 -0800896 exception_return
Antonio Nino Diaz594811b2019-01-31 11:58:00 +0000897
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100898endfunc el3_exit