Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 1 | /* |
Antonio Nino Diaz | 0e402d3 | 2019-01-30 16:01:49 +0000 | [diff] [blame] | 2 | * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 3 | * |
dp-arm | fa3cf0b | 2017-05-03 09:38:09 +0100 | [diff] [blame] | 4 | * SPDX-License-Identifier: BSD-3-Clause |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 5 | */ |
| 6 | |
Antonio Nino Diaz | e0f9063 | 2018-12-14 00:18:21 +0000 | [diff] [blame] | 7 | #include <platform_def.h> |
| 8 | |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 9 | #include <arch.h> |
Dan Handley | 714a0d2 | 2014-04-09 13:13:04 +0100 | [diff] [blame] | 10 | #include <asm_macros.S> |
Antonio Nino Diaz | e0f9063 | 2018-12-14 00:18:21 +0000 | [diff] [blame] | 11 | #include <bl31/ea_handle.h> |
| 12 | #include <bl31/interrupt_mgmt.h> |
| 13 | #include <common/runtime_svc.h> |
Dan Handley | 2bd4ef2 | 2014-04-09 13:14:54 +0100 | [diff] [blame] | 14 | #include <context.h> |
Antonio Nino Diaz | e0f9063 | 2018-12-14 00:18:21 +0000 | [diff] [blame] | 15 | #include <lib/el3_runtime/cpu_data.h> |
| 16 | #include <lib/smccc.h> |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 17 | |
| 18 | .globl runtime_exceptions |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 19 | |
Dimitris Papastamos | 446f7f1 | 2017-11-30 14:53:53 +0000 | [diff] [blame] | 20 | .globl sync_exception_sp_el0 |
| 21 | .globl irq_sp_el0 |
| 22 | .globl fiq_sp_el0 |
| 23 | .globl serror_sp_el0 |
| 24 | |
| 25 | .globl sync_exception_sp_elx |
| 26 | .globl irq_sp_elx |
| 27 | .globl fiq_sp_elx |
| 28 | .globl serror_sp_elx |
| 29 | |
| 30 | .globl sync_exception_aarch64 |
| 31 | .globl irq_aarch64 |
| 32 | .globl fiq_aarch64 |
| 33 | .globl serror_aarch64 |
| 34 | |
| 35 | .globl sync_exception_aarch32 |
| 36 | .globl irq_aarch32 |
| 37 | .globl fiq_aarch32 |
| 38 | .globl serror_aarch32 |
| 39 | |
Jeenu Viswambharan | 96c7df0 | 2017-11-30 12:54:15 +0000 | [diff] [blame] | 40 | /* |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 41 | * Macro that prepares entry to EL3 upon taking an exception. |
| 42 | * |
| 43 | * With RAS_EXTENSION, this macro synchronizes pending errors with an ESB |
| 44 | * instruction. When an error is thus synchronized, the handling is |
| 45 | * delegated to platform EA handler. |
| 46 | * |
| 47 | * Without RAS_EXTENSION, this macro just saves x30, and unmasks |
| 48 | * Asynchronous External Aborts. |
| 49 | */ |
| 50 | .macro check_and_unmask_ea |
| 51 | #if RAS_EXTENSION |
| 52 | /* Synchronize pending External Aborts */ |
| 53 | esb |
| 54 | |
| 55 | /* Unmask the SError interrupt */ |
| 56 | msr daifclr, #DAIF_ABT_BIT |
| 57 | |
| 58 | /* |
| 59 | * Explicitly save x30 so as to free up a register and to enable |
| 60 | * branching |
| 61 | */ |
| 62 | str x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] |
| 63 | |
| 64 | /* Check for SErrors synchronized by the ESB instruction */ |
| 65 | mrs x30, DISR_EL1 |
| 66 | tbz x30, #DISR_A_BIT, 1f |
| 67 | |
| 68 | /* Save GP registers and restore them afterwards */ |
| 69 | bl save_gp_registers |
Alexei Fedorov | 503bbf3 | 2019-08-13 15:17:53 +0100 | [diff] [blame] | 70 | |
| 71 | /* |
| 72 | * If Secure Cycle Counter is not disabled in MDCR_EL3 |
| 73 | * when ARMv8.5-PMU is implemented, save PMCR_EL0 and |
| 74 | * disable all event counters and cycle counter. |
| 75 | */ |
| 76 | bl save_pmcr_disable_pmu |
| 77 | |
Jeenu Viswambharan | e86a247 | 2018-07-05 15:24:45 +0100 | [diff] [blame] | 78 | bl handle_lower_el_ea_esb |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 79 | bl restore_gp_registers |
| 80 | |
| 81 | 1: |
| 82 | #else |
| 83 | /* Unmask the SError interrupt */ |
| 84 | msr daifclr, #DAIF_ABT_BIT |
| 85 | |
| 86 | str x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] |
| 87 | #endif |
| 88 | .endm |
| 89 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 90 | /* --------------------------------------------------------------------- |
| 91 | * This macro handles Synchronous exceptions. |
| 92 | * Only SMC exceptions are supported. |
| 93 | * --------------------------------------------------------------------- |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 94 | */ |
| 95 | .macro handle_sync_exception |
dp-arm | 3cac786 | 2016-09-19 11:18:44 +0100 | [diff] [blame] | 96 | #if ENABLE_RUNTIME_INSTRUMENTATION |
dp-arm | 3cac786 | 2016-09-19 11:18:44 +0100 | [diff] [blame] | 97 | /* |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 98 | * Read the timestamp value and store it in per-cpu data. The value |
| 99 | * will be extracted from per-cpu data by the C level SMC handler and |
| 100 | * saved to the PMF timestamp region. |
dp-arm | 3cac786 | 2016-09-19 11:18:44 +0100 | [diff] [blame] | 101 | */ |
| 102 | mrs x30, cntpct_el0 |
| 103 | str x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] |
| 104 | mrs x29, tpidr_el3 |
| 105 | str x30, [x29, #CPU_DATA_PMF_TS0_OFFSET] |
| 106 | ldr x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] |
| 107 | #endif |
| 108 | |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 109 | mrs x30, esr_el3 |
| 110 | ubfx x30, x30, #ESR_EC_SHIFT, #ESR_EC_LENGTH |
| 111 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 112 | /* Handle SMC exceptions separately from other synchronous exceptions */ |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 113 | cmp x30, #EC_AARCH32_SMC |
| 114 | b.eq smc_handler32 |
| 115 | |
| 116 | cmp x30, #EC_AARCH64_SMC |
| 117 | b.eq smc_handler64 |
| 118 | |
Jeenu Viswambharan | e86a247 | 2018-07-05 15:24:45 +0100 | [diff] [blame] | 119 | /* Synchronous exceptions other than the above are assumed to be EA */ |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 120 | ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] |
Jeenu Viswambharan | e86a247 | 2018-07-05 15:24:45 +0100 | [diff] [blame] | 121 | b enter_lower_el_sync_ea |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 122 | .endm |
| 123 | |
| 124 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 125 | /* --------------------------------------------------------------------- |
| 126 | * This macro handles FIQ or IRQ interrupts i.e. EL3, S-EL1 and NS |
| 127 | * interrupts. |
| 128 | * --------------------------------------------------------------------- |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 129 | */ |
| 130 | .macro handle_interrupt_exception label |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 131 | |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 132 | bl save_gp_registers |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 133 | |
Alexei Fedorov | 503bbf3 | 2019-08-13 15:17:53 +0100 | [diff] [blame] | 134 | /* |
| 135 | * If Secure Cycle Counter is not disabled in MDCR_EL3 |
| 136 | * when ARMv8.5-PMU is implemented, save PMCR_EL0 and |
| 137 | * disable all event counters and cycle counter. |
| 138 | */ |
| 139 | bl save_pmcr_disable_pmu |
| 140 | |
Antonio Nino Diaz | 25cda67 | 2019-02-19 11:53:51 +0000 | [diff] [blame] | 141 | /* Save ARMv8.3-PAuth registers and load firmware key */ |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 142 | #if CTX_INCLUDE_PAUTH_REGS |
| 143 | bl pauth_context_save |
| 144 | #endif |
Antonio Nino Diaz | 25cda67 | 2019-02-19 11:53:51 +0000 | [diff] [blame] | 145 | #if ENABLE_PAUTH |
| 146 | bl pauth_load_bl_apiakey |
| 147 | #endif |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 148 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 149 | /* Save the EL3 system registers needed to return from this exception */ |
Achin Gupta | 979992e | 2015-05-13 17:57:18 +0100 | [diff] [blame] | 150 | mrs x0, spsr_el3 |
| 151 | mrs x1, elr_el3 |
| 152 | stp x0, x1, [sp, #CTX_EL3STATE_OFFSET + CTX_SPSR_EL3] |
| 153 | |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 154 | /* Switch to the runtime stack i.e. SP_EL0 */ |
| 155 | ldr x2, [sp, #CTX_EL3STATE_OFFSET + CTX_RUNTIME_SP] |
| 156 | mov x20, sp |
| 157 | msr spsel, #0 |
| 158 | mov sp, x2 |
| 159 | |
| 160 | /* |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 161 | * Find out whether this is a valid interrupt type. |
| 162 | * If the interrupt controller reports a spurious interrupt then return |
| 163 | * to where we came from. |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 164 | */ |
Dan Handley | 701fea7 | 2014-05-27 16:17:21 +0100 | [diff] [blame] | 165 | bl plat_ic_get_pending_interrupt_type |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 166 | cmp x0, #INTR_TYPE_INVAL |
| 167 | b.eq interrupt_exit_\label |
| 168 | |
| 169 | /* |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 170 | * Get the registered handler for this interrupt type. |
| 171 | * A NULL return value could be 'cause of the following conditions: |
Achin Gupta | 979992e | 2015-05-13 17:57:18 +0100 | [diff] [blame] | 172 | * |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 173 | * a. An interrupt of a type was routed correctly but a handler for its |
| 174 | * type was not registered. |
Achin Gupta | 979992e | 2015-05-13 17:57:18 +0100 | [diff] [blame] | 175 | * |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 176 | * b. An interrupt of a type was not routed correctly so a handler for |
| 177 | * its type was not registered. |
Achin Gupta | 979992e | 2015-05-13 17:57:18 +0100 | [diff] [blame] | 178 | * |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 179 | * c. An interrupt of a type was routed correctly to EL3, but was |
| 180 | * deasserted before its pending state could be read. Another |
| 181 | * interrupt of a different type pended at the same time and its |
| 182 | * type was reported as pending instead. However, a handler for this |
| 183 | * type was not registered. |
Achin Gupta | 979992e | 2015-05-13 17:57:18 +0100 | [diff] [blame] | 184 | * |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 185 | * a. and b. can only happen due to a programming error. The |
| 186 | * occurrence of c. could be beyond the control of Trusted Firmware. |
| 187 | * It makes sense to return from this exception instead of reporting an |
| 188 | * error. |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 189 | */ |
| 190 | bl get_interrupt_type_handler |
Achin Gupta | 979992e | 2015-05-13 17:57:18 +0100 | [diff] [blame] | 191 | cbz x0, interrupt_exit_\label |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 192 | mov x21, x0 |
| 193 | |
| 194 | mov x0, #INTR_ID_UNAVAILABLE |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 195 | |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 196 | /* Set the current security state in the 'flags' parameter */ |
| 197 | mrs x2, scr_el3 |
| 198 | ubfx x1, x2, #0, #1 |
| 199 | |
| 200 | /* Restore the reference to the 'handle' i.e. SP_EL3 */ |
| 201 | mov x2, x20 |
| 202 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 203 | /* x3 will point to a cookie (not used now) */ |
Soby Mathew | 799f0ab | 2014-05-27 16:54:31 +0100 | [diff] [blame] | 204 | mov x3, xzr |
| 205 | |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 206 | /* Call the interrupt type handler */ |
| 207 | blr x21 |
| 208 | |
| 209 | interrupt_exit_\label: |
| 210 | /* Return from exception, possibly in a different security state */ |
| 211 | b el3_exit |
| 212 | |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 213 | .endm |
| 214 | |
| 215 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 216 | vector_base runtime_exceptions |
| 217 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 218 | /* --------------------------------------------------------------------- |
| 219 | * Current EL with SP_EL0 : 0x0 - 0x200 |
| 220 | * --------------------------------------------------------------------- |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 221 | */ |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 222 | vector_entry sync_exception_sp_el0 |
Justin Chadwell | 83e0488 | 2019-08-20 11:01:52 +0100 | [diff] [blame] | 223 | #ifdef MONITOR_TRAPS |
| 224 | stp x29, x30, [sp, #-16]! |
| 225 | |
| 226 | mrs x30, esr_el3 |
| 227 | ubfx x30, x30, #ESR_EC_SHIFT, #ESR_EC_LENGTH |
| 228 | |
| 229 | /* Check for BRK */ |
| 230 | cmp x30, #EC_BRK |
| 231 | b.eq brk_handler |
| 232 | |
| 233 | ldp x29, x30, [sp], #16 |
| 234 | #endif /* MONITOR_TRAPS */ |
| 235 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 236 | /* We don't expect any synchronous exceptions from EL3 */ |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 237 | b report_unhandled_exception |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 238 | end_vector_entry sync_exception_sp_el0 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 239 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 240 | vector_entry irq_sp_el0 |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 241 | /* |
| 242 | * EL3 code is non-reentrant. Any asynchronous exception is a serious |
| 243 | * error. Loop infinitely. |
| 244 | */ |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 245 | b report_unhandled_interrupt |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 246 | end_vector_entry irq_sp_el0 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 247 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 248 | |
| 249 | vector_entry fiq_sp_el0 |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 250 | b report_unhandled_interrupt |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 251 | end_vector_entry fiq_sp_el0 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 252 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 253 | |
| 254 | vector_entry serror_sp_el0 |
Jeenu Viswambharan | 911fcc9 | 2018-07-06 16:50:06 +0100 | [diff] [blame] | 255 | no_ret plat_handle_el3_ea |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 256 | end_vector_entry serror_sp_el0 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 257 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 258 | /* --------------------------------------------------------------------- |
| 259 | * Current EL with SP_ELx: 0x200 - 0x400 |
| 260 | * --------------------------------------------------------------------- |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 261 | */ |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 262 | vector_entry sync_exception_sp_elx |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 263 | /* |
| 264 | * This exception will trigger if anything went wrong during a previous |
| 265 | * exception entry or exit or while handling an earlier unexpected |
| 266 | * synchronous exception. There is a high probability that SP_EL3 is |
| 267 | * corrupted. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 268 | */ |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 269 | b report_unhandled_exception |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 270 | end_vector_entry sync_exception_sp_elx |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 271 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 272 | vector_entry irq_sp_elx |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 273 | b report_unhandled_interrupt |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 274 | end_vector_entry irq_sp_elx |
Jeenu Viswambharan | a7934d6 | 2014-02-07 15:53:18 +0000 | [diff] [blame] | 275 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 276 | vector_entry fiq_sp_elx |
Julius Werner | 67ebde7 | 2017-07-27 14:59:34 -0700 | [diff] [blame] | 277 | b report_unhandled_interrupt |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 278 | end_vector_entry fiq_sp_elx |
Jeenu Viswambharan | a7934d6 | 2014-02-07 15:53:18 +0000 | [diff] [blame] | 279 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 280 | vector_entry serror_sp_elx |
Jeenu Viswambharan | 911fcc9 | 2018-07-06 16:50:06 +0100 | [diff] [blame] | 281 | no_ret plat_handle_el3_ea |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 282 | end_vector_entry serror_sp_elx |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 283 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 284 | /* --------------------------------------------------------------------- |
Sandrine Bailleux | 046cd3f | 2014-08-06 11:27:23 +0100 | [diff] [blame] | 285 | * Lower EL using AArch64 : 0x400 - 0x600 |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 286 | * --------------------------------------------------------------------- |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 287 | */ |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 288 | vector_entry sync_exception_aarch64 |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 289 | /* |
| 290 | * This exception vector will be the entry point for SMCs and traps |
| 291 | * that are unhandled at lower ELs most commonly. SP_EL3 should point |
| 292 | * to a valid cpu context where the general purpose and system register |
| 293 | * state can be saved. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 294 | */ |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 295 | check_and_unmask_ea |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 296 | handle_sync_exception |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 297 | end_vector_entry sync_exception_aarch64 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 298 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 299 | vector_entry irq_aarch64 |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 300 | check_and_unmask_ea |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 301 | handle_interrupt_exception irq_aarch64 |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 302 | end_vector_entry irq_aarch64 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 303 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 304 | vector_entry fiq_aarch64 |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 305 | check_and_unmask_ea |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 306 | handle_interrupt_exception fiq_aarch64 |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 307 | end_vector_entry fiq_aarch64 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 308 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 309 | vector_entry serror_aarch64 |
Jeenu Viswambharan | 96c7df0 | 2017-11-30 12:54:15 +0000 | [diff] [blame] | 310 | msr daifclr, #DAIF_ABT_BIT |
Jeenu Viswambharan | e86a247 | 2018-07-05 15:24:45 +0100 | [diff] [blame] | 311 | b enter_lower_el_async_ea |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 312 | end_vector_entry serror_aarch64 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 313 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 314 | /* --------------------------------------------------------------------- |
Sandrine Bailleux | 046cd3f | 2014-08-06 11:27:23 +0100 | [diff] [blame] | 315 | * Lower EL using AArch32 : 0x600 - 0x800 |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 316 | * --------------------------------------------------------------------- |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 317 | */ |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 318 | vector_entry sync_exception_aarch32 |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 319 | /* |
| 320 | * This exception vector will be the entry point for SMCs and traps |
| 321 | * that are unhandled at lower ELs most commonly. SP_EL3 should point |
| 322 | * to a valid cpu context where the general purpose and system register |
| 323 | * state can be saved. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 324 | */ |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 325 | check_and_unmask_ea |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 326 | handle_sync_exception |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 327 | end_vector_entry sync_exception_aarch32 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 328 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 329 | vector_entry irq_aarch32 |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 330 | check_and_unmask_ea |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 331 | handle_interrupt_exception irq_aarch32 |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 332 | end_vector_entry irq_aarch32 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 333 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 334 | vector_entry fiq_aarch32 |
Jeenu Viswambharan | 9a7ce2f | 2018-04-04 16:07:11 +0100 | [diff] [blame] | 335 | check_and_unmask_ea |
Achin Gupta | 9cf2bb7 | 2014-05-09 11:07:09 +0100 | [diff] [blame] | 336 | handle_interrupt_exception fiq_aarch32 |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 337 | end_vector_entry fiq_aarch32 |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 338 | |
Sandrine Bailleux | 9e6ad6c | 2016-05-24 16:56:03 +0100 | [diff] [blame] | 339 | vector_entry serror_aarch32 |
Jeenu Viswambharan | 96c7df0 | 2017-11-30 12:54:15 +0000 | [diff] [blame] | 340 | msr daifclr, #DAIF_ABT_BIT |
Jeenu Viswambharan | e86a247 | 2018-07-05 15:24:45 +0100 | [diff] [blame] | 341 | b enter_lower_el_async_ea |
Roberto Vargas | 95f30ab | 2018-04-17 11:31:43 +0100 | [diff] [blame] | 342 | end_vector_entry serror_aarch32 |
Jeenu Viswambharan | a7934d6 | 2014-02-07 15:53:18 +0000 | [diff] [blame] | 343 | |
Justin Chadwell | 83e0488 | 2019-08-20 11:01:52 +0100 | [diff] [blame] | 344 | #ifdef MONITOR_TRAPS |
| 345 | .section .rodata.brk_string, "aS" |
| 346 | brk_location: |
| 347 | .asciz "Error at instruction 0x" |
| 348 | brk_message: |
| 349 | .asciz "Unexpected BRK instruction with value 0x" |
| 350 | #endif /* MONITOR_TRAPS */ |
| 351 | |
Antonio Nino Diaz | 35c8cfc | 2018-04-23 15:43:29 +0100 | [diff] [blame] | 352 | /* --------------------------------------------------------------------- |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 353 | * The following code handles secure monitor calls. |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 354 | * Depending upon the execution state from where the SMC has been |
| 355 | * invoked, it frees some general purpose registers to perform the |
| 356 | * remaining tasks. They involve finding the runtime service handler |
| 357 | * that is the target of the SMC & switching to runtime stacks (SP_EL0) |
| 358 | * before calling the handler. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 359 | * |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 360 | * Note that x30 has been explicitly saved and can be used here |
| 361 | * --------------------------------------------------------------------- |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 362 | */ |
Andrew Thoelke | 38bde41 | 2014-03-18 13:46:55 +0000 | [diff] [blame] | 363 | func smc_handler |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 364 | smc_handler32: |
| 365 | /* Check whether aarch32 issued an SMC64 */ |
| 366 | tbnz x0, #FUNCID_CC_SHIFT, smc_prohibited |
| 367 | |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 368 | smc_handler64: |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 369 | /* NOTE: The code below must preserve x0-x4 */ |
| 370 | |
| 371 | /* Save general purpose registers */ |
| 372 | bl save_gp_registers |
| 373 | |
Alexei Fedorov | 503bbf3 | 2019-08-13 15:17:53 +0100 | [diff] [blame] | 374 | /* |
| 375 | * If Secure Cycle Counter is not disabled in MDCR_EL3 |
| 376 | * when ARMv8.5-PMU is implemented, save PMCR_EL0 and |
| 377 | * disable all event counters and cycle counter. |
| 378 | */ |
| 379 | bl save_pmcr_disable_pmu |
| 380 | |
Antonio Nino Diaz | 25cda67 | 2019-02-19 11:53:51 +0000 | [diff] [blame] | 381 | /* Save ARMv8.3-PAuth registers and load firmware key */ |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 382 | #if CTX_INCLUDE_PAUTH_REGS |
| 383 | bl pauth_context_save |
| 384 | #endif |
Antonio Nino Diaz | 25cda67 | 2019-02-19 11:53:51 +0000 | [diff] [blame] | 385 | #if ENABLE_PAUTH |
| 386 | bl pauth_load_bl_apiakey |
| 387 | #endif |
Antonio Nino Diaz | 594811b | 2019-01-31 11:58:00 +0000 | [diff] [blame] | 388 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 389 | /* |
| 390 | * Populate the parameters for the SMC handler. |
| 391 | * We already have x0-x4 in place. x5 will point to a cookie (not used |
| 392 | * now). x6 will point to the context structure (SP_EL3) and x7 will |
Dimitris Papastamos | 0415951 | 2018-01-22 11:53:04 +0000 | [diff] [blame] | 393 | * contain flags we need to pass to the handler. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 394 | */ |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 395 | mov x5, xzr |
| 396 | mov x6, sp |
| 397 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 398 | /* |
Antonio Nino Diaz | 35c8cfc | 2018-04-23 15:43:29 +0100 | [diff] [blame] | 399 | * Restore the saved C runtime stack value which will become the new |
| 400 | * SP_EL0 i.e. EL3 runtime stack. It was saved in the 'cpu_context' |
| 401 | * structure prior to the last ERET from EL3. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 402 | */ |
Antonio Nino Diaz | 35c8cfc | 2018-04-23 15:43:29 +0100 | [diff] [blame] | 403 | ldr x12, [x6, #CTX_EL3STATE_OFFSET + CTX_RUNTIME_SP] |
| 404 | |
| 405 | /* Switch to SP_EL0 */ |
| 406 | msr spsel, #0 |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 407 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 408 | /* |
| 409 | * Save the SPSR_EL3, ELR_EL3, & SCR_EL3 in case there is a world |
| 410 | * switch during SMC handling. |
| 411 | * TODO: Revisit if all system registers can be saved later. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 412 | */ |
| 413 | mrs x16, spsr_el3 |
| 414 | mrs x17, elr_el3 |
| 415 | mrs x18, scr_el3 |
| 416 | stp x16, x17, [x6, #CTX_EL3STATE_OFFSET + CTX_SPSR_EL3] |
Achin Gupta | e1aa516 | 2014-06-26 09:58:52 +0100 | [diff] [blame] | 417 | str x18, [x6, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3] |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 418 | |
| 419 | /* Copy SCR_EL3.NS bit to the flag to indicate caller's security */ |
| 420 | bfi x7, x18, #0, #1 |
| 421 | |
| 422 | mov sp, x12 |
| 423 | |
Madhukar Pappireddy | d87233a | 2019-05-08 15:41:41 -0500 | [diff] [blame] | 424 | /* Get the unique owning entity number */ |
| 425 | ubfx x16, x0, #FUNCID_OEN_SHIFT, #FUNCID_OEN_WIDTH |
| 426 | ubfx x15, x0, #FUNCID_TYPE_SHIFT, #FUNCID_TYPE_WIDTH |
| 427 | orr x16, x16, x15, lsl #FUNCID_OEN_WIDTH |
| 428 | |
| 429 | /* Load descriptor index from array of indices */ |
| 430 | adr x14, rt_svc_descs_indices |
| 431 | ldrb w15, [x14, x16] |
| 432 | |
| 433 | /* Any index greater than 127 is invalid. Check bit 7. */ |
| 434 | tbnz w15, 7, smc_unknown |
| 435 | |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 436 | /* |
Madhukar Pappireddy | d87233a | 2019-05-08 15:41:41 -0500 | [diff] [blame] | 437 | * Get the descriptor using the index |
| 438 | * x11 = (base + off), w15 = index |
| 439 | * |
| 440 | * handler = (base + off) + (index << log2(size)) |
| 441 | */ |
| 442 | adr x11, (__RT_SVC_DESCS_START__ + RT_SVC_DESC_HANDLE) |
| 443 | lsl w10, w15, #RT_SVC_SIZE_LOG2 |
| 444 | ldr x15, [x11, w10, uxtw] |
| 445 | |
| 446 | /* |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 447 | * Call the Secure Monitor Call handler and then drop directly into |
| 448 | * el3_exit() which will program any remaining architectural state |
| 449 | * prior to issuing the ERET to the desired lower EL. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 450 | */ |
| 451 | #if DEBUG |
| 452 | cbz x15, rt_svc_fw_critical_error |
| 453 | #endif |
| 454 | blr x15 |
| 455 | |
Yatharth Kochar | 6c0566c | 2015-10-02 17:56:48 +0100 | [diff] [blame] | 456 | b el3_exit |
Achin Gupta | 4f6ad66 | 2013-10-25 09:08:21 +0100 | [diff] [blame] | 457 | |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 458 | smc_unknown: |
| 459 | /* |
Madhukar Pappireddy | d87233a | 2019-05-08 15:41:41 -0500 | [diff] [blame] | 460 | * Unknown SMC call. Populate return value with SMC_UNK and call |
| 461 | * el3_exit() which will restore the remaining architectural state |
| 462 | * i.e., SYS, GP and PAuth registers(if any) prior to issuing the ERET |
| 463 | * to the desired lower EL. |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 464 | */ |
Antonio Nino Diaz | e4794b7 | 2018-02-14 14:22:29 +0000 | [diff] [blame] | 465 | mov x0, #SMC_UNK |
Madhukar Pappireddy | d87233a | 2019-05-08 15:41:41 -0500 | [diff] [blame] | 466 | str x0, [x6, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] |
| 467 | b el3_exit |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 468 | |
| 469 | smc_prohibited: |
Soby Mathew | 6c5192a | 2014-04-30 15:36:37 +0100 | [diff] [blame] | 470 | ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] |
Antonio Nino Diaz | e4794b7 | 2018-02-14 14:22:29 +0000 | [diff] [blame] | 471 | mov x0, #SMC_UNK |
Jeenu Viswambharan | caa8493 | 2014-02-06 10:36:15 +0000 | [diff] [blame] | 472 | eret |
| 473 | |
| 474 | rt_svc_fw_critical_error: |
Douglas Raillard | 0980eed | 2016-11-09 17:48:27 +0000 | [diff] [blame] | 475 | /* Switch to SP_ELx */ |
| 476 | msr spsel, #1 |
Jeenu Viswambharan | 68aef10 | 2016-11-30 15:21:11 +0000 | [diff] [blame] | 477 | no_ret report_unhandled_exception |
Kévin Petit | a877c25 | 2015-03-24 14:03:57 +0000 | [diff] [blame] | 478 | endfunc smc_handler |
Justin Chadwell | 83e0488 | 2019-08-20 11:01:52 +0100 | [diff] [blame] | 479 | |
| 480 | /* --------------------------------------------------------------------- |
| 481 | * The following code handles exceptions caused by BRK instructions. |
| 482 | * Following a BRK instruction, the only real valid cause of action is |
| 483 | * to print some information and panic, as the code that caused it is |
| 484 | * likely in an inconsistent internal state. |
| 485 | * |
| 486 | * This is initially intended to be used in conjunction with |
| 487 | * __builtin_trap. |
| 488 | * --------------------------------------------------------------------- |
| 489 | */ |
| 490 | #ifdef MONITOR_TRAPS |
| 491 | func brk_handler |
| 492 | /* Extract the ISS */ |
| 493 | mrs x10, esr_el3 |
| 494 | ubfx x10, x10, #ESR_ISS_SHIFT, #ESR_ISS_LENGTH |
| 495 | |
| 496 | /* Ensure the console is initialized */ |
| 497 | bl plat_crash_console_init |
| 498 | |
| 499 | adr x4, brk_location |
| 500 | bl asm_print_str |
| 501 | mrs x4, elr_el3 |
| 502 | bl asm_print_hex |
| 503 | bl asm_print_newline |
| 504 | |
| 505 | adr x4, brk_message |
| 506 | bl asm_print_str |
| 507 | mov x4, x10 |
| 508 | mov x5, #28 |
| 509 | bl asm_print_hex_bits |
| 510 | bl asm_print_newline |
| 511 | |
| 512 | no_ret plat_panic_handler |
| 513 | endfunc brk_handler |
| 514 | #endif /* MONITOR_TRAPS */ |