blob: 389c0869ab4c2712979b83afb14de799bdf15c57 [file] [log] [blame]
Achin Gupta9ac63c52014-01-16 12:08:03 +00001/*
Rohit Mathew3dc3cad2022-11-11 18:45:11 +00002 * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved.
Achin Gupta9ac63c52014-01-16 12:08:03 +00003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Achin Gupta9ac63c52014-01-16 12:08:03 +00005 */
6
Dan Handley2bd4ef22014-04-09 13:14:54 +01007#include <arch.h>
Andrew Thoelke38bde412014-03-18 13:46:55 +00008#include <asm_macros.S>
Jan Dabrosfa015982019-12-02 13:30:03 +01009#include <assert_macros.S>
Dan Handley2bd4ef22014-04-09 13:14:54 +010010#include <context.h>
Manish V Badarkhee07e8082020-07-23 12:43:25 +010011#include <el3_common_macros.S>
Achin Gupta9ac63c52014-01-16 12:08:03 +000012
Yatharth Kochar6c0566c2015-10-02 17:56:48 +010013 .global el1_sysregs_context_save
14 .global el1_sysregs_context_restore
15#if CTX_INCLUDE_FPREGS
16 .global fpregs_context_save
17 .global fpregs_context_restore
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +000018#endif /* CTX_INCLUDE_FPREGS */
Daniel Boulby95fb1aa2022-01-19 11:20:05 +000019 .global prepare_el3_entry
Alexei Fedorovf41355c2019-09-13 14:11:59 +010020 .global restore_gp_pmcr_pauth_regs
Manish V Badarkhee07e8082020-07-23 12:43:25 +010021 .global save_and_update_ptw_el1_sys_regs
Yatharth Kochar6c0566c2015-10-02 17:56:48 +010022 .global el3_exit
23
Max Shvetsovbdf502d2020-02-25 13:56:19 +000024
Alexei Fedorovf41355c2019-09-13 14:11:59 +010025/* ------------------------------------------------------------------
26 * The following function strictly follows the AArch64 PCS to use
27 * x9-x17 (temporary caller-saved registers) to save EL1 system
28 * register context. It assumes that 'x0' is pointing to a
29 * 'el1_sys_regs' structure where the register context will be saved.
30 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +000031 */
Andrew Thoelke38bde412014-03-18 13:46:55 +000032func el1_sysregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +000033
34 mrs x9, spsr_el1
35 mrs x10, elr_el1
36 stp x9, x10, [x0, #CTX_SPSR_EL1]
37
Manish V Badarkhee07e8082020-07-23 12:43:25 +010038#if !ERRATA_SPECULATIVE_AT
Achin Gupta9ac63c52014-01-16 12:08:03 +000039 mrs x15, sctlr_el1
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +010040 mrs x16, tcr_el1
Achin Gupta9ac63c52014-01-16 12:08:03 +000041 stp x15, x16, [x0, #CTX_SCTLR_EL1]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +000042#endif /* ERRATA_SPECULATIVE_AT */
Achin Gupta9ac63c52014-01-16 12:08:03 +000043
44 mrs x17, cpacr_el1
45 mrs x9, csselr_el1
46 stp x17, x9, [x0, #CTX_CPACR_EL1]
47
48 mrs x10, sp_el1
49 mrs x11, esr_el1
50 stp x10, x11, [x0, #CTX_SP_EL1]
51
52 mrs x12, ttbr0_el1
53 mrs x13, ttbr1_el1
54 stp x12, x13, [x0, #CTX_TTBR0_EL1]
55
56 mrs x14, mair_el1
57 mrs x15, amair_el1
58 stp x14, x15, [x0, #CTX_MAIR_EL1]
59
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +010060 mrs x16, actlr_el1
Achin Gupta9ac63c52014-01-16 12:08:03 +000061 mrs x17, tpidr_el1
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +010062 stp x16, x17, [x0, #CTX_ACTLR_EL1]
Achin Gupta9ac63c52014-01-16 12:08:03 +000063
64 mrs x9, tpidr_el0
65 mrs x10, tpidrro_el0
66 stp x9, x10, [x0, #CTX_TPIDR_EL0]
67
Achin Gupta9ac63c52014-01-16 12:08:03 +000068 mrs x13, par_el1
69 mrs x14, far_el1
70 stp x13, x14, [x0, #CTX_PAR_EL1]
71
72 mrs x15, afsr0_el1
73 mrs x16, afsr1_el1
74 stp x15, x16, [x0, #CTX_AFSR0_EL1]
75
76 mrs x17, contextidr_el1
77 mrs x9, vbar_el1
78 stp x17, x9, [x0, #CTX_CONTEXTIDR_EL1]
79
Soby Mathewd75d2ba2016-05-17 14:01:32 +010080 /* Save AArch32 system registers if the build has instructed so */
81#if CTX_INCLUDE_AARCH32_REGS
82 mrs x11, spsr_abt
83 mrs x12, spsr_und
84 stp x11, x12, [x0, #CTX_SPSR_ABT]
85
86 mrs x13, spsr_irq
87 mrs x14, spsr_fiq
88 stp x13, x14, [x0, #CTX_SPSR_IRQ]
89
90 mrs x15, dacr32_el2
91 mrs x16, ifsr32_el2
92 stp x15, x16, [x0, #CTX_DACR32_EL2]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +000093#endif /* CTX_INCLUDE_AARCH32_REGS */
Soby Mathewd75d2ba2016-05-17 14:01:32 +010094
Jeenu Viswambharand1b60152014-05-12 15:28:47 +010095 /* Save NS timer registers if the build has instructed so */
96#if NS_TIMER_SWITCH
Achin Gupta9ac63c52014-01-16 12:08:03 +000097 mrs x10, cntp_ctl_el0
98 mrs x11, cntp_cval_el0
99 stp x10, x11, [x0, #CTX_CNTP_CTL_EL0]
100
101 mrs x12, cntv_ctl_el0
102 mrs x13, cntv_cval_el0
103 stp x12, x13, [x0, #CTX_CNTV_CTL_EL0]
104
105 mrs x14, cntkctl_el1
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100106 str x14, [x0, #CTX_CNTKCTL_EL1]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000107#endif /* NS_TIMER_SWITCH */
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100108
Justin Chadwell1c7c13a2019-07-18 14:25:33 +0100109 /* Save MTE system registers if the build has instructed so */
110#if CTX_INCLUDE_MTE_REGS
111 mrs x15, TFSRE0_EL1
112 mrs x16, TFSR_EL1
113 stp x15, x16, [x0, #CTX_TFSRE0_EL1]
114
115 mrs x9, RGSR_EL1
116 mrs x10, GCR_EL1
117 stp x9, x10, [x0, #CTX_RGSR_EL1]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000118#endif /* CTX_INCLUDE_MTE_REGS */
Justin Chadwell1c7c13a2019-07-18 14:25:33 +0100119
Achin Gupta9ac63c52014-01-16 12:08:03 +0000120 ret
Kévin Petita877c252015-03-24 14:03:57 +0000121endfunc el1_sysregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000122
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100123/* ------------------------------------------------------------------
124 * The following function strictly follows the AArch64 PCS to use
125 * x9-x17 (temporary caller-saved registers) to restore EL1 system
126 * register context. It assumes that 'x0' is pointing to a
127 * 'el1_sys_regs' structure from where the register context will be
128 * restored
129 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000130 */
Andrew Thoelke38bde412014-03-18 13:46:55 +0000131func el1_sysregs_context_restore
Achin Gupta9ac63c52014-01-16 12:08:03 +0000132
133 ldp x9, x10, [x0, #CTX_SPSR_EL1]
134 msr spsr_el1, x9
135 msr elr_el1, x10
136
Manish V Badarkhee07e8082020-07-23 12:43:25 +0100137#if !ERRATA_SPECULATIVE_AT
Manish V Badarkhed73c1ba2020-07-28 07:12:56 +0100138 ldp x15, x16, [x0, #CTX_SCTLR_EL1]
139 msr sctlr_el1, x15
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100140 msr tcr_el1, x16
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000141#endif /* ERRATA_SPECULATIVE_AT */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000142
143 ldp x17, x9, [x0, #CTX_CPACR_EL1]
144 msr cpacr_el1, x17
145 msr csselr_el1, x9
146
147 ldp x10, x11, [x0, #CTX_SP_EL1]
148 msr sp_el1, x10
149 msr esr_el1, x11
150
151 ldp x12, x13, [x0, #CTX_TTBR0_EL1]
152 msr ttbr0_el1, x12
153 msr ttbr1_el1, x13
154
155 ldp x14, x15, [x0, #CTX_MAIR_EL1]
156 msr mair_el1, x14
157 msr amair_el1, x15
158
Manish V Badarkhe2b0ee972020-07-28 07:22:30 +0100159 ldp x16, x17, [x0, #CTX_ACTLR_EL1]
160 msr actlr_el1, x16
Manish V Badarkhed73c1ba2020-07-28 07:12:56 +0100161 msr tpidr_el1, x17
Achin Gupta9ac63c52014-01-16 12:08:03 +0000162
163 ldp x9, x10, [x0, #CTX_TPIDR_EL0]
164 msr tpidr_el0, x9
165 msr tpidrro_el0, x10
166
Achin Gupta9ac63c52014-01-16 12:08:03 +0000167 ldp x13, x14, [x0, #CTX_PAR_EL1]
168 msr par_el1, x13
169 msr far_el1, x14
170
171 ldp x15, x16, [x0, #CTX_AFSR0_EL1]
172 msr afsr0_el1, x15
173 msr afsr1_el1, x16
174
175 ldp x17, x9, [x0, #CTX_CONTEXTIDR_EL1]
176 msr contextidr_el1, x17
177 msr vbar_el1, x9
178
Soby Mathewd75d2ba2016-05-17 14:01:32 +0100179 /* Restore AArch32 system registers if the build has instructed so */
180#if CTX_INCLUDE_AARCH32_REGS
181 ldp x11, x12, [x0, #CTX_SPSR_ABT]
182 msr spsr_abt, x11
183 msr spsr_und, x12
184
185 ldp x13, x14, [x0, #CTX_SPSR_IRQ]
186 msr spsr_irq, x13
187 msr spsr_fiq, x14
188
189 ldp x15, x16, [x0, #CTX_DACR32_EL2]
190 msr dacr32_el2, x15
191 msr ifsr32_el2, x16
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000192#endif /* CTX_INCLUDE_AARCH32_REGS */
193
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100194 /* Restore NS timer registers if the build has instructed so */
195#if NS_TIMER_SWITCH
Achin Gupta9ac63c52014-01-16 12:08:03 +0000196 ldp x10, x11, [x0, #CTX_CNTP_CTL_EL0]
197 msr cntp_ctl_el0, x10
198 msr cntp_cval_el0, x11
199
200 ldp x12, x13, [x0, #CTX_CNTV_CTL_EL0]
201 msr cntv_ctl_el0, x12
202 msr cntv_cval_el0, x13
203
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100204 ldr x14, [x0, #CTX_CNTKCTL_EL1]
Achin Gupta9ac63c52014-01-16 12:08:03 +0000205 msr cntkctl_el1, x14
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000206#endif /* NS_TIMER_SWITCH */
207
Justin Chadwell1c7c13a2019-07-18 14:25:33 +0100208 /* Restore MTE system registers if the build has instructed so */
209#if CTX_INCLUDE_MTE_REGS
210 ldp x11, x12, [x0, #CTX_TFSRE0_EL1]
211 msr TFSRE0_EL1, x11
212 msr TFSR_EL1, x12
213
214 ldp x13, x14, [x0, #CTX_RGSR_EL1]
215 msr RGSR_EL1, x13
216 msr GCR_EL1, x14
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000217#endif /* CTX_INCLUDE_MTE_REGS */
Jeenu Viswambharand1b60152014-05-12 15:28:47 +0100218
Achin Gupta9ac63c52014-01-16 12:08:03 +0000219 /* No explict ISB required here as ERET covers it */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000220 ret
Kévin Petita877c252015-03-24 14:03:57 +0000221endfunc el1_sysregs_context_restore
Achin Gupta9ac63c52014-01-16 12:08:03 +0000222
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100223/* ------------------------------------------------------------------
224 * The following function follows the aapcs_64 strictly to use
225 * x9-x17 (temporary caller-saved registers according to AArch64 PCS)
226 * to save floating point register context. It assumes that 'x0' is
227 * pointing to a 'fp_regs' structure where the register context will
Achin Gupta9ac63c52014-01-16 12:08:03 +0000228 * be saved.
229 *
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100230 * Access to VFP registers will trap if CPTR_EL3.TFP is set.
231 * However currently we don't use VFP registers nor set traps in
232 * Trusted Firmware, and assume it's cleared.
Achin Gupta9ac63c52014-01-16 12:08:03 +0000233 *
234 * TODO: Revisit when VFP is used in secure world
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100235 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000236 */
Juan Castillo258e94f2014-06-25 17:26:36 +0100237#if CTX_INCLUDE_FPREGS
Andrew Thoelke38bde412014-03-18 13:46:55 +0000238func fpregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000239 stp q0, q1, [x0, #CTX_FP_Q0]
240 stp q2, q3, [x0, #CTX_FP_Q2]
241 stp q4, q5, [x0, #CTX_FP_Q4]
242 stp q6, q7, [x0, #CTX_FP_Q6]
243 stp q8, q9, [x0, #CTX_FP_Q8]
244 stp q10, q11, [x0, #CTX_FP_Q10]
245 stp q12, q13, [x0, #CTX_FP_Q12]
246 stp q14, q15, [x0, #CTX_FP_Q14]
247 stp q16, q17, [x0, #CTX_FP_Q16]
248 stp q18, q19, [x0, #CTX_FP_Q18]
249 stp q20, q21, [x0, #CTX_FP_Q20]
250 stp q22, q23, [x0, #CTX_FP_Q22]
251 stp q24, q25, [x0, #CTX_FP_Q24]
252 stp q26, q27, [x0, #CTX_FP_Q26]
253 stp q28, q29, [x0, #CTX_FP_Q28]
254 stp q30, q31, [x0, #CTX_FP_Q30]
255
256 mrs x9, fpsr
257 str x9, [x0, #CTX_FP_FPSR]
258
259 mrs x10, fpcr
260 str x10, [x0, #CTX_FP_FPCR]
261
David Cunadod1a1fd42017-10-20 11:30:57 +0100262#if CTX_INCLUDE_AARCH32_REGS
263 mrs x11, fpexc32_el2
264 str x11, [x0, #CTX_FP_FPEXC32_EL2]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000265#endif /* CTX_INCLUDE_AARCH32_REGS */
Achin Gupta9ac63c52014-01-16 12:08:03 +0000266 ret
Kévin Petita877c252015-03-24 14:03:57 +0000267endfunc fpregs_context_save
Achin Gupta9ac63c52014-01-16 12:08:03 +0000268
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100269/* ------------------------------------------------------------------
270 * The following function follows the aapcs_64 strictly to use x9-x17
271 * (temporary caller-saved registers according to AArch64 PCS) to
272 * restore floating point register context. It assumes that 'x0' is
273 * pointing to a 'fp_regs' structure from where the register context
Achin Gupta9ac63c52014-01-16 12:08:03 +0000274 * will be restored.
275 *
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100276 * Access to VFP registers will trap if CPTR_EL3.TFP is set.
277 * However currently we don't use VFP registers nor set traps in
278 * Trusted Firmware, and assume it's cleared.
Achin Gupta9ac63c52014-01-16 12:08:03 +0000279 *
280 * TODO: Revisit when VFP is used in secure world
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100281 * ------------------------------------------------------------------
Achin Gupta9ac63c52014-01-16 12:08:03 +0000282 */
Andrew Thoelke38bde412014-03-18 13:46:55 +0000283func fpregs_context_restore
Achin Gupta9ac63c52014-01-16 12:08:03 +0000284 ldp q0, q1, [x0, #CTX_FP_Q0]
285 ldp q2, q3, [x0, #CTX_FP_Q2]
286 ldp q4, q5, [x0, #CTX_FP_Q4]
287 ldp q6, q7, [x0, #CTX_FP_Q6]
288 ldp q8, q9, [x0, #CTX_FP_Q8]
289 ldp q10, q11, [x0, #CTX_FP_Q10]
290 ldp q12, q13, [x0, #CTX_FP_Q12]
291 ldp q14, q15, [x0, #CTX_FP_Q14]
292 ldp q16, q17, [x0, #CTX_FP_Q16]
293 ldp q18, q19, [x0, #CTX_FP_Q18]
294 ldp q20, q21, [x0, #CTX_FP_Q20]
295 ldp q22, q23, [x0, #CTX_FP_Q22]
296 ldp q24, q25, [x0, #CTX_FP_Q24]
297 ldp q26, q27, [x0, #CTX_FP_Q26]
298 ldp q28, q29, [x0, #CTX_FP_Q28]
299 ldp q30, q31, [x0, #CTX_FP_Q30]
300
301 ldr x9, [x0, #CTX_FP_FPSR]
302 msr fpsr, x9
303
Soby Mathewe77e1162015-12-03 09:42:50 +0000304 ldr x10, [x0, #CTX_FP_FPCR]
Achin Gupta9ac63c52014-01-16 12:08:03 +0000305 msr fpcr, x10
306
David Cunadod1a1fd42017-10-20 11:30:57 +0100307#if CTX_INCLUDE_AARCH32_REGS
308 ldr x11, [x0, #CTX_FP_FPEXC32_EL2]
309 msr fpexc32_el2, x11
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000310#endif /* CTX_INCLUDE_AARCH32_REGS */
311
Achin Gupta9ac63c52014-01-16 12:08:03 +0000312 /*
313 * No explict ISB required here as ERET to
Sandrine Bailleuxf4119ec2015-12-17 13:58:58 +0000314 * switch to secure EL1 or non-secure world
Achin Gupta9ac63c52014-01-16 12:08:03 +0000315 * covers it
316 */
317
318 ret
Kévin Petita877c252015-03-24 14:03:57 +0000319endfunc fpregs_context_restore
Juan Castillo258e94f2014-06-25 17:26:36 +0100320#endif /* CTX_INCLUDE_FPREGS */
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100321
Daniel Boulby928747f2021-05-25 18:09:34 +0100322 /*
Manish Pandey62d532a2022-11-17 15:47:05 +0000323 * Set SCR_EL3.EA bit to enable SErrors at EL3
324 */
325 .macro enable_serror_at_el3
326 mrs x8, scr_el3
327 orr x8, x8, #SCR_EA_BIT
328 msr scr_el3, x8
329 .endm
330
331 /*
Daniel Boulby928747f2021-05-25 18:09:34 +0100332 * Set the PSTATE bits not set when the exception was taken as
333 * described in the AArch64.TakeException() pseudocode function
334 * in ARM DDI 0487F.c page J1-7635 to a default value.
335 */
336 .macro set_unset_pstate_bits
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000337 /*
338 * If Data Independent Timing (DIT) functionality is implemented,
339 * always enable DIT in EL3
340 */
Daniel Boulby928747f2021-05-25 18:09:34 +0100341#if ENABLE_FEAT_DIT
Andre Przywara1f55c412023-01-26 16:47:52 +0000342#if ENABLE_FEAT_DIT == 2
343 mrs x8, id_aa64pfr0_el1
344 and x8, x8, #(ID_AA64PFR0_DIT_MASK << ID_AA64PFR0_DIT_SHIFT)
345 cbz x8, 1f
346#endif
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000347 mov x8, #DIT_BIT
348 msr DIT, x8
Andre Przywara1f55c412023-01-26 16:47:52 +00003491:
Daniel Boulby928747f2021-05-25 18:09:34 +0100350#endif /* ENABLE_FEAT_DIT */
351 .endm /* set_unset_pstate_bits */
352
Arvind Ram Prakashab28d4b2023-10-11 12:10:56 -0500353/*-------------------------------------------------------------------------
354 * This macro checks the ENABLE_FEAT_MPAM state, performs ID register
355 * check to see if the platform supports MPAM extension and restores MPAM3
356 * register value if it is FEAT_STATE_ENABLED/FEAT_STATE_CHECKED.
357 *
358 * This is particularly more complicated because we can't check
359 * if the platform supports MPAM by looking for status of a particular bit
360 * in the MDCR_EL3 or CPTR_EL3 register like other extensions.
361 * ------------------------------------------------------------------------
362 */
363
364 .macro restore_mpam3_el3
365#if ENABLE_FEAT_MPAM
366#if ENABLE_FEAT_MPAM == 2
367
368 mrs x8, id_aa64pfr0_el1
369 lsr x8, x8, #(ID_AA64PFR0_MPAM_SHIFT)
370 and x8, x8, #(ID_AA64PFR0_MPAM_MASK)
371 mrs x7, id_aa64pfr1_el1
372 lsr x7, x7, #(ID_AA64PFR1_MPAM_FRAC_SHIFT)
373 and x7, x7, #(ID_AA64PFR1_MPAM_FRAC_MASK)
374 orr x7, x7, x8
375 cbz x7, no_mpam
376#endif
377 /* -----------------------------------------------------------
378 * Restore MPAM3_EL3 register as per context state
379 * Currently we only enable MPAM for NS world and trap to EL3
380 * for MPAM access in lower ELs of Secure and Realm world
Arvind Ram Prakashb5d95592023-11-08 12:28:30 -0600381 * x9 holds address of the per_world context
Arvind Ram Prakashab28d4b2023-10-11 12:10:56 -0500382 * -----------------------------------------------------------
383 */
Arvind Ram Prakashb5d95592023-11-08 12:28:30 -0600384
385 ldr x17, [x9, #CTX_MPAM3_EL3]
Arvind Ram Prakashab28d4b2023-10-11 12:10:56 -0500386 msr S3_6_C10_C5_0, x17 /* mpam3_el3 */
387
388no_mpam:
389#endif
390 .endm /* restore_mpam3_el3 */
391
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100392/* ------------------------------------------------------------------
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000393 * The following macro is used to save and restore all the general
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100394 * purpose and ARMv8.3-PAuth (if enabled) registers.
Jayanth Dodderi Chidanand4ec78ad2022-09-19 23:32:08 +0100395 * It also checks if the Secure Cycle Counter (PMCCNTR_EL0)
396 * is disabled in EL3/Secure (ARMv8.5-PMU), wherein PMCCNTR_EL0
397 * needs not to be saved/restored during world switch.
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100398 *
399 * Ideally we would only save and restore the callee saved registers
400 * when a world switch occurs but that type of implementation is more
401 * complex. So currently we will always save and restore these
402 * registers on entry and exit of EL3.
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100403 * clobbers: x18
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100404 * ------------------------------------------------------------------
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100405 */
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000406 .macro save_gp_pmcr_pauth_regs
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100407 stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
408 stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
409 stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
410 stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
411 stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
412 stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
413 stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
414 stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
415 stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
416 stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
417 stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
418 stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
419 stp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
420 stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
421 stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
422 mrs x18, sp_el0
423 str x18, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0]
Boyan Karatotev05504ba2023-02-15 13:21:50 +0000424
425 /* PMUv3 is presumed to be always present */
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100426 mrs x9, pmcr_el0
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100427 str x9, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0]
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100428 /* Disable cycle counter when event counting is prohibited */
Boyan Karatoteved85cf72022-12-06 09:03:42 +0000429 orr x9, x9, #PMCR_EL0_DP_BIT
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100430 msr pmcr_el0, x9
431 isb
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100432#if CTX_INCLUDE_PAUTH_REGS
433 /* ----------------------------------------------------------
434 * Save the ARMv8.3-PAuth keys as they are not banked
435 * by exception level
436 * ----------------------------------------------------------
437 */
438 add x19, sp, #CTX_PAUTH_REGS_OFFSET
439
440 mrs x20, APIAKeyLo_EL1 /* x21:x20 = APIAKey */
441 mrs x21, APIAKeyHi_EL1
442 mrs x22, APIBKeyLo_EL1 /* x23:x22 = APIBKey */
443 mrs x23, APIBKeyHi_EL1
444 mrs x24, APDAKeyLo_EL1 /* x25:x24 = APDAKey */
445 mrs x25, APDAKeyHi_EL1
446 mrs x26, APDBKeyLo_EL1 /* x27:x26 = APDBKey */
447 mrs x27, APDBKeyHi_EL1
448 mrs x28, APGAKeyLo_EL1 /* x29:x28 = APGAKey */
449 mrs x29, APGAKeyHi_EL1
450
451 stp x20, x21, [x19, #CTX_PACIAKEY_LO]
452 stp x22, x23, [x19, #CTX_PACIBKEY_LO]
453 stp x24, x25, [x19, #CTX_PACDAKEY_LO]
454 stp x26, x27, [x19, #CTX_PACDBKEY_LO]
455 stp x28, x29, [x19, #CTX_PACGAKEY_LO]
456#endif /* CTX_INCLUDE_PAUTH_REGS */
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000457 .endm /* save_gp_pmcr_pauth_regs */
458
459/* -----------------------------------------------------------------
Daniel Boulby928747f2021-05-25 18:09:34 +0100460 * This function saves the context and sets the PSTATE to a known
461 * state, preparing entry to el3.
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000462 * Save all the general purpose and ARMv8.3-PAuth (if enabled)
463 * registers.
Daniel Boulby928747f2021-05-25 18:09:34 +0100464 * Then set any of the PSTATE bits that are not set by hardware
465 * according to the Aarch64.TakeException pseudocode in the Arm
466 * Architecture Reference Manual to a default value for EL3.
467 * clobbers: x17
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000468 * -----------------------------------------------------------------
469 */
470func prepare_el3_entry
471 save_gp_pmcr_pauth_regs
Manish Pandey62d532a2022-11-17 15:47:05 +0000472 enable_serror_at_el3
Daniel Boulby928747f2021-05-25 18:09:34 +0100473 /*
474 * Set the PSTATE bits not described in the Aarch64.TakeException
475 * pseudocode to their default values.
476 */
477 set_unset_pstate_bits
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100478 ret
Daniel Boulby95fb1aa2022-01-19 11:20:05 +0000479endfunc prepare_el3_entry
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100480
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100481/* ------------------------------------------------------------------
482 * This function restores ARMv8.3-PAuth (if enabled) and all general
483 * purpose registers except x30 from the CPU context.
484 * x30 register must be explicitly restored by the caller.
485 * ------------------------------------------------------------------
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000486 */
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100487func restore_gp_pmcr_pauth_regs
488#if CTX_INCLUDE_PAUTH_REGS
489 /* Restore the ARMv8.3 PAuth keys */
490 add x10, sp, #CTX_PAUTH_REGS_OFFSET
491
492 ldp x0, x1, [x10, #CTX_PACIAKEY_LO] /* x1:x0 = APIAKey */
493 ldp x2, x3, [x10, #CTX_PACIBKEY_LO] /* x3:x2 = APIBKey */
494 ldp x4, x5, [x10, #CTX_PACDAKEY_LO] /* x5:x4 = APDAKey */
495 ldp x6, x7, [x10, #CTX_PACDBKEY_LO] /* x7:x6 = APDBKey */
496 ldp x8, x9, [x10, #CTX_PACGAKEY_LO] /* x9:x8 = APGAKey */
497
498 msr APIAKeyLo_EL1, x0
499 msr APIAKeyHi_EL1, x1
500 msr APIBKeyLo_EL1, x2
501 msr APIBKeyHi_EL1, x3
502 msr APDAKeyLo_EL1, x4
503 msr APDAKeyHi_EL1, x5
504 msr APDBKeyLo_EL1, x6
505 msr APDBKeyHi_EL1, x7
506 msr APGAKeyLo_EL1, x8
507 msr APGAKeyHi_EL1, x9
508#endif /* CTX_INCLUDE_PAUTH_REGS */
Boyan Karatotev05504ba2023-02-15 13:21:50 +0000509
510 /* PMUv3 is presumed to be always present */
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100511 ldr x0, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0]
512 msr pmcr_el0, x0
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100513 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
514 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100515 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
516 ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
517 ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
518 ldp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
519 ldp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
520 ldp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000521 ldp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100522 ldp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
523 ldp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
524 ldp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
525 ldp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
526 ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000527 ldr x28, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0]
528 msr sp_el0, x28
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100529 ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000530 ret
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100531endfunc restore_gp_pmcr_pauth_regs
Jeenu Viswambharan23d05a82017-11-29 16:59:34 +0000532
Manish V Badarkhee07e8082020-07-23 12:43:25 +0100533/*
534 * In case of ERRATA_SPECULATIVE_AT, save SCTLR_EL1 and TCR_EL1
535 * registers and update EL1 registers to disable stage1 and stage2
536 * page table walk
537 */
538func save_and_update_ptw_el1_sys_regs
539 /* ----------------------------------------------------------
540 * Save only sctlr_el1 and tcr_el1 registers
541 * ----------------------------------------------------------
542 */
543 mrs x29, sctlr_el1
544 str x29, [sp, #(CTX_EL1_SYSREGS_OFFSET + CTX_SCTLR_EL1)]
545 mrs x29, tcr_el1
546 str x29, [sp, #(CTX_EL1_SYSREGS_OFFSET + CTX_TCR_EL1)]
547
548 /* ------------------------------------------------------------
549 * Must follow below order in order to disable page table
550 * walk for lower ELs (EL1 and EL0). First step ensures that
551 * page table walk is disabled for stage1 and second step
552 * ensures that page table walker should use TCR_EL1.EPDx
553 * bits to perform address translation. ISB ensures that CPU
554 * does these 2 steps in order.
555 *
556 * 1. Update TCR_EL1.EPDx bits to disable page table walk by
557 * stage1.
558 * 2. Enable MMU bit to avoid identity mapping via stage2
559 * and force TCR_EL1.EPDx to be used by the page table
560 * walker.
561 * ------------------------------------------------------------
562 */
563 orr x29, x29, #(TCR_EPD0_BIT)
564 orr x29, x29, #(TCR_EPD1_BIT)
565 msr tcr_el1, x29
566 isb
567 mrs x29, sctlr_el1
568 orr x29, x29, #SCTLR_M_BIT
569 msr sctlr_el1, x29
570 isb
571
572 ret
573endfunc save_and_update_ptw_el1_sys_regs
574
Elizabeth Ho4fc00d22023-07-18 14:10:25 +0100575/* -----------------------------------------------------------------
576* The below macro returns the address of the per_world context for
577* the security state, retrieved through "get_security_state" macro.
578* The per_world context address is returned in the register argument.
579* Clobbers: x9, x10
580* ------------------------------------------------------------------
581*/
582
583.macro get_per_world_context _reg:req
584 ldr x10, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3]
585 get_security_state x9, x10
Jayanth Dodderi Chidanand56aa3822023-12-11 11:22:02 +0000586 mov_imm x10, (CTX_PERWORLD_EL3STATE_END - CTX_CPTR_EL3)
Elizabeth Ho4fc00d22023-07-18 14:10:25 +0100587 mul x9, x9, x10
588 adrp x10, per_world_context
589 add x10, x10, :lo12:per_world_context
590 add x9, x9, x10
591 mov \_reg, x9
592.endm
593
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100594/* ------------------------------------------------------------------
595 * This routine assumes that the SP_EL3 is pointing to a valid
596 * context structure from where the gp regs and other special
597 * registers can be retrieved.
598 * ------------------------------------------------------------------
Antonio Nino Diaz13adfb12019-01-30 20:41:31 +0000599 */
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100600func el3_exit
Jan Dabrosfa015982019-12-02 13:30:03 +0100601#if ENABLE_ASSERTIONS
602 /* el3_exit assumes SP_EL0 on entry */
603 mrs x17, spsel
604 cmp x17, #MODE_SP_EL0
605 ASM_ASSERT(eq)
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000606#endif /* ENABLE_ASSERTIONS */
Jan Dabrosfa015982019-12-02 13:30:03 +0100607
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100608 /* ----------------------------------------------------------
609 * Save the current SP_EL0 i.e. the EL3 runtime stack which
610 * will be used for handling the next SMC.
611 * Then switch to SP_EL3.
612 * ----------------------------------------------------------
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100613 */
614 mov x17, sp
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100615 msr spsel, #MODE_SP_ELX
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100616 str x17, [sp, #CTX_EL3STATE_OFFSET + CTX_RUNTIME_SP]
617
Max Shvetsovc4502772021-03-22 11:59:37 +0000618 /* ----------------------------------------------------------
Arunachalam Ganapathycac7d162021-07-08 09:35:57 +0100619 * Restore CPTR_EL3.
Max Shvetsovc4502772021-03-22 11:59:37 +0000620 * ZCR is only restored if SVE is supported and enabled.
621 * Synchronization is required before zcr_el3 is addressed.
622 * ----------------------------------------------------------
623 */
Elizabeth Ho4fc00d22023-07-18 14:10:25 +0100624
625 /* The address of the per_world context is stored in x9 */
626 get_per_world_context x9
627
628 ldp x19, x20, [x9, #CTX_CPTR_EL3]
Max Shvetsovc4502772021-03-22 11:59:37 +0000629 msr cptr_el3, x19
630
Boyan Karatotev8ae58f02023-04-20 11:00:50 +0100631#if IMAGE_BL31
Max Shvetsovc4502772021-03-22 11:59:37 +0000632 ands x19, x19, #CPTR_EZ_BIT
633 beq sve_not_enabled
634
635 isb
636 msr S3_6_C1_C2_0, x20 /* zcr_el3 */
637sve_not_enabled:
Arvind Ram Prakashab28d4b2023-10-11 12:10:56 -0500638
639 restore_mpam3_el3
640
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000641#endif /* IMAGE_BL31 */
Max Shvetsovc4502772021-03-22 11:59:37 +0000642
Dimitris Papastamosba51d9e2018-05-16 11:36:14 +0100643#if IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100644 /* ----------------------------------------------------------
645 * Restore mitigation state as it was on entry to EL3
646 * ----------------------------------------------------------
647 */
Dimitris Papastamosba51d9e2018-05-16 11:36:14 +0100648 ldr x17, [sp, #CTX_CVE_2018_3639_OFFSET + CTX_CVE_2018_3639_DISABLE]
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100649 cbz x17, 1f
Dimitris Papastamosba51d9e2018-05-16 11:36:14 +0100650 blr x17
Antonio Nino Diaz13adfb12019-01-30 20:41:31 +00006511:
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000652#endif /* IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 */
653
Manish Pandey6b5721f2023-06-26 17:46:14 +0100654#if IMAGE_BL31
655 synchronize_errors
656#endif /* IMAGE_BL31 */
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000657
Manish Pandey53bc59a2022-11-17 14:43:15 +0000658 /* ----------------------------------------------------------
659 * Restore SPSR_EL3, ELR_EL3 and SCR_EL3 prior to ERET
660 * ----------------------------------------------------------
661 */
662 ldr x18, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3]
663 ldp x16, x17, [sp, #CTX_EL3STATE_OFFSET + CTX_SPSR_EL3]
664 msr scr_el3, x18
665 msr spsr_el3, x16
666 msr elr_el3, x17
667
668 restore_ptw_el1_sys_regs
669
670 /* ----------------------------------------------------------
671 * Restore general purpose (including x30), PMCR_EL0 and
672 * ARMv8.3-PAuth registers.
673 * Exit EL3 via ERET to a lower exception level.
674 * ----------------------------------------------------------
675 */
676 bl restore_gp_pmcr_pauth_regs
677 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
678
Madhukar Pappireddyfba25722020-07-24 03:27:12 -0500679#ifdef IMAGE_BL31
Manish Pandey07952fb2023-05-25 13:46:14 +0100680 /* Clear the EL3 flag as we are exiting el3 */
681 str xzr, [sp, #CTX_EL3STATE_OFFSET + CTX_NESTED_EA_FLAG]
Jayanth Dodderi Chidanand72b69b82022-01-26 17:14:43 +0000682#endif /* IMAGE_BL31 */
683
Anthony Steinhauser0f7e6012020-01-07 15:44:06 -0800684 exception_return
Antonio Nino Diaz594811b2019-01-31 11:58:00 +0000685
Yatharth Kochar6c0566c2015-10-02 17:56:48 +0100686endfunc el3_exit