blob: 65f2df5235fdfbec4d1d6b2d54b754850926a217 [file] [log] [blame]
Jeenu Viswambharan10a67272017-09-22 08:32:10 +01001/*
2 * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7/*
8 * Exception handlers at EL3, their priority levels, and management.
9 */
10
11#include <assert.h>
12#include <cpu_data.h>
13#include <debug.h>
14#include <ehf.h>
Jeenu Viswambharan6c6f24d2017-10-04 12:21:34 +010015#include <gic_common.h>
Jeenu Viswambharan10a67272017-09-22 08:32:10 +010016#include <interrupt_mgmt.h>
17#include <platform.h>
Jeenu Viswambharan6c6f24d2017-10-04 12:21:34 +010018#include <pubsub_events.h>
Jeenu Viswambharan10a67272017-09-22 08:32:10 +010019
20/* Output EHF logs as verbose */
21#define EHF_LOG(...) VERBOSE("EHF: " __VA_ARGS__)
22
23#define EHF_INVALID_IDX (-1)
24
25/* For a valid handler, return the actual function pointer; otherwise, 0. */
26#define RAW_HANDLER(h) \
27 ((ehf_handler_t) ((h & _EHF_PRI_VALID) ? (h & ~_EHF_PRI_VALID) : 0))
28
29#define PRI_BIT(idx) (((ehf_pri_bits_t) 1) << idx)
30
31/*
32 * Convert index into secure priority using the platform-defined priority bits
33 * field.
34 */
35#define IDX_TO_PRI(idx) \
36 ((idx << (7 - exception_data.pri_bits)) & 0x7f)
37
38/* Check whether a given index is valid */
39#define IS_IDX_VALID(idx) \
40 ((exception_data.ehf_priorities[idx].ehf_handler & _EHF_PRI_VALID) != 0)
41
42/* Returns whether given priority is in secure priority range */
43#define IS_PRI_SECURE(pri) ((pri & 0x80) == 0)
44
45/* To be defined by the platform */
46extern const ehf_priorities_t exception_data;
47
48/* Translate priority to the index in the priority array */
49static int pri_to_idx(unsigned int priority)
50{
51 int idx;
52
53 idx = EHF_PRI_TO_IDX(priority, exception_data.pri_bits);
54 assert((idx >= 0) && (idx < exception_data.num_priorities));
55 assert(IS_IDX_VALID(idx));
56
57 return idx;
58}
59
60/* Return whether there are outstanding priority activation */
61static int has_valid_pri_activations(pe_exc_data_t *pe_data)
62{
63 return pe_data->active_pri_bits != 0;
64}
65
66static pe_exc_data_t *this_cpu_data(void)
67{
68 return &get_cpu_data(ehf_data);
69}
70
71/*
72 * Return the current priority index of this CPU. If no priority is active,
73 * return EHF_INVALID_IDX.
74 */
75static int get_pe_highest_active_idx(pe_exc_data_t *pe_data)
76{
77 if (!has_valid_pri_activations(pe_data))
78 return EHF_INVALID_IDX;
79
80 /* Current priority is the right-most bit */
81 return __builtin_ctz(pe_data->active_pri_bits);
82}
83
84/*
85 * Mark priority active by setting the corresponding bit in active_pri_bits and
86 * programming the priority mask.
87 *
88 * This API is to be used as part of delegating to lower ELs other than for
89 * interrupts; e.g. while handling synchronous exceptions.
90 *
91 * This API is expected to be invoked before restoring context (Secure or
92 * Non-secure) in preparation for the respective dispatch.
93 */
94void ehf_activate_priority(unsigned int priority)
95{
96 int idx, cur_pri_idx;
97 unsigned int old_mask, run_pri;
98 pe_exc_data_t *pe_data = this_cpu_data();
99
100 /*
101 * Query interrupt controller for the running priority, or idle priority
102 * if no interrupts are being handled. The requested priority must be
103 * less (higher priority) than the active running priority.
104 */
105 run_pri = plat_ic_get_running_priority();
106 if (priority >= run_pri) {
107 ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
108 run_pri, priority);
109 panic();
110 }
111
112 /*
113 * If there were priority activations already, the requested priority
114 * must be less (higher priority) than the current highest priority
115 * activation so far.
116 */
117 cur_pri_idx = get_pe_highest_active_idx(pe_data);
118 idx = pri_to_idx(priority);
119 if ((cur_pri_idx != EHF_INVALID_IDX) && (idx >= cur_pri_idx)) {
120 ERROR("Activation priority mismatch: req=0x%x current=0x%x\n",
121 priority, IDX_TO_PRI(cur_pri_idx));
122 panic();
123 }
124
125 /* Set the bit corresponding to the requested priority */
126 pe_data->active_pri_bits |= PRI_BIT(idx);
127
128 /*
129 * Program priority mask for the activated level. Check that the new
130 * priority mask is setting a higher priority level than the existing
131 * mask.
132 */
133 old_mask = plat_ic_set_priority_mask(priority);
134 if (priority >= old_mask) {
135 ERROR("Requested priority (0x%x) lower than Priority Mask (0x%x)\n",
136 priority, old_mask);
137 panic();
138 }
139
140 /*
141 * If this is the first activation, save the priority mask. This will be
142 * restored after the last deactivation.
143 */
144 if (cur_pri_idx == EHF_INVALID_IDX)
145 pe_data->init_pri_mask = old_mask;
146
147 EHF_LOG("activate prio=%d\n", get_pe_highest_active_idx(pe_data));
148}
149
150/*
151 * Mark priority inactive by clearing the corresponding bit in active_pri_bits,
152 * and programming the priority mask.
153 *
154 * This API is expected to be used as part of delegating to to lower ELs other
155 * than for interrupts; e.g. while handling synchronous exceptions.
156 *
157 * This API is expected to be invoked after saving context (Secure or
158 * Non-secure), having concluded the respective dispatch.
159 */
160void ehf_deactivate_priority(unsigned int priority)
161{
162 int idx, cur_pri_idx;
163 pe_exc_data_t *pe_data = this_cpu_data();
164 unsigned int old_mask, run_pri;
165
166 /*
167 * Query interrupt controller for the running priority, or idle priority
168 * if no interrupts are being handled. The requested priority must be
169 * less (higher priority) than the active running priority.
170 */
171 run_pri = plat_ic_get_running_priority();
172 if (priority >= run_pri) {
173 ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
174 run_pri, priority);
175 panic();
176 }
177
178 /*
179 * Deactivation is allowed only when there are priority activations, and
180 * the deactivation priority level must match the current activated
181 * priority.
182 */
183 cur_pri_idx = get_pe_highest_active_idx(pe_data);
184 idx = pri_to_idx(priority);
185 if ((cur_pri_idx == EHF_INVALID_IDX) || (idx != cur_pri_idx)) {
186 ERROR("Deactivation priority mismatch: req=0x%x current=0x%x\n",
187 priority, IDX_TO_PRI(cur_pri_idx));
188 panic();
189 }
190
191 /* Clear bit corresponding to highest priority */
192 pe_data->active_pri_bits &= (pe_data->active_pri_bits - 1);
193
194 /*
195 * Restore priority mask corresponding to the next priority, or the
196 * one stashed earlier if there are no more to deactivate.
197 */
198 idx = get_pe_highest_active_idx(pe_data);
199 if (idx == EHF_INVALID_IDX)
200 old_mask = plat_ic_set_priority_mask(pe_data->init_pri_mask);
201 else
202 old_mask = plat_ic_set_priority_mask(priority);
203
204 if (old_mask >= priority) {
205 ERROR("Deactivation priority (0x%x) lower than Priority Mask (0x%x)\n",
206 priority, old_mask);
207 panic();
208 }
209
210 EHF_LOG("deactivate prio=%d\n", get_pe_highest_active_idx(pe_data));
211}
212
213/*
Jeenu Viswambharan6c6f24d2017-10-04 12:21:34 +0100214 * After leaving Non-secure world, stash current Non-secure Priority Mask, and
215 * set Priority Mask to the highest Non-secure priority so that Non-secure
216 * interrupts cannot preempt Secure execution.
217 *
218 * If the current running priority is in the secure range, or if there are
219 * outstanding priority activations, this function does nothing.
220 *
221 * This function subscribes to the 'cm_exited_normal_world' event published by
222 * the Context Management Library.
223 */
224static void *ehf_exited_normal_world(const void *arg)
225{
226 unsigned int run_pri;
227 pe_exc_data_t *pe_data = this_cpu_data();
228
229 /* If the running priority is in the secure range, do nothing */
230 run_pri = plat_ic_get_running_priority();
231 if (IS_PRI_SECURE(run_pri))
232 return 0;
233
234 /* Do nothing if there are explicit activations */
235 if (has_valid_pri_activations(pe_data))
236 return 0;
237
238 assert(pe_data->ns_pri_mask == 0);
239
240 pe_data->ns_pri_mask =
241 plat_ic_set_priority_mask(GIC_HIGHEST_NS_PRIORITY);
242
243 /* The previous Priority Mask is not expected to be in secure range */
244 if (IS_PRI_SECURE(pe_data->ns_pri_mask)) {
245 ERROR("Priority Mask (0x%x) already in secure range\n",
246 pe_data->ns_pri_mask);
247 panic();
248 }
249
250 EHF_LOG("Priority Mask: 0x%x => 0x%x\n", pe_data->ns_pri_mask,
251 GIC_HIGHEST_NS_PRIORITY);
252
253 return 0;
254}
255
256/*
257 * Conclude Secure execution and prepare for return to Non-secure world. Restore
258 * the Non-secure Priority Mask previously stashed upon leaving Non-secure
259 * world.
260 *
261 * If there the current running priority is in the secure range, or if there are
262 * outstanding priority activations, this function does nothing.
263 *
264 * This function subscribes to the 'cm_entering_normal_world' event published by
265 * the Context Management Library.
266 */
267static void *ehf_entering_normal_world(const void *arg)
268{
269 unsigned int old_pmr, run_pri;
270 pe_exc_data_t *pe_data = this_cpu_data();
271
272 /* If the running priority is in the secure range, do nothing */
273 run_pri = plat_ic_get_running_priority();
274 if (IS_PRI_SECURE(run_pri))
275 return 0;
276
277 /*
278 * If there are explicit activations, do nothing. The Priority Mask will
279 * be restored upon the last deactivation.
280 */
281 if (has_valid_pri_activations(pe_data))
282 return 0;
283
284 /* Do nothing if we don't have a valid Priority Mask to restore */
285 if (pe_data->ns_pri_mask == 0)
286 return 0;
287
288 old_pmr = plat_ic_set_priority_mask(pe_data->ns_pri_mask);
289
290 /*
291 * When exiting secure world, the current Priority Mask must be
292 * GIC_HIGHEST_NS_PRIORITY (as set during entry), or the Non-secure
293 * priority mask set upon calling ehf_allow_ns_preemption()
294 */
295 if ((old_pmr != GIC_HIGHEST_NS_PRIORITY) &&
296 (old_pmr != pe_data->ns_pri_mask)) {
297 ERROR("Invalid Priority Mask (0x%x) restored\n", old_pmr);
298 panic();
299 }
300
301 EHF_LOG("Priority Mask: 0x%x => 0x%x\n", old_pmr, pe_data->ns_pri_mask);
302
303 pe_data->ns_pri_mask = 0;
304
305 return 0;
306}
307
308/*
309 * Program Priority Mask to the original Non-secure priority such that
310 * Non-secure interrupts may preempt Secure execution, viz. during Yielding SMC
311 * calls.
312 *
313 * This API is expected to be invoked before delegating a yielding SMC to Secure
314 * EL1. I.e. within the window of secure execution after Non-secure context is
315 * saved (after entry into EL3) and Secure context is restored (before entering
316 * Secure EL1).
317 */
318void ehf_allow_ns_preemption(void)
319{
320 unsigned int old_pmr __unused;
321 pe_exc_data_t *pe_data = this_cpu_data();
322
323 /*
324 * We should have been notified earlier of entering secure world, and
325 * therefore have stashed the Non-secure priority mask.
326 */
327 assert(pe_data->ns_pri_mask != 0);
328
329 /* Make sure no priority levels are active when requesting this */
330 if (has_valid_pri_activations(pe_data)) {
331 ERROR("PE %lx has priority activations: 0x%x\n",
332 read_mpidr_el1(), pe_data->active_pri_bits);
333 panic();
334 }
335
336 old_pmr = plat_ic_set_priority_mask(pe_data->ns_pri_mask);
337
338 EHF_LOG("Priority Mask: 0x%x => 0x%x\n", old_pmr, pe_data->ns_pri_mask);
339
340 pe_data->ns_pri_mask = 0;
341}
342
343/*
344 * Return whether Secure execution has explicitly allowed Non-secure interrupts
345 * to preempt itself, viz. during Yielding SMC calls.
346 */
347unsigned int ehf_is_ns_preemption_allowed(void)
348{
349 unsigned int run_pri;
350 pe_exc_data_t *pe_data = this_cpu_data();
351
352 /* If running priority is in secure range, return false */
353 run_pri = plat_ic_get_running_priority();
354 if (IS_PRI_SECURE(run_pri))
355 return 0;
356
357 /*
358 * If Non-secure preemption was permitted by calling
359 * ehf_allow_ns_preemption() earlier:
360 *
361 * - There wouldn't have been priority activations;
362 * - We would have cleared the stashed the Non-secure Priority Mask.
363 */
364 if (has_valid_pri_activations(pe_data))
365 return 0;
366 if (pe_data->ns_pri_mask != 0)
367 return 0;
368
369 return 1;
370}
371
372/*
Jeenu Viswambharan10a67272017-09-22 08:32:10 +0100373 * Top-level EL3 interrupt handler.
374 */
375static uint64_t ehf_el3_interrupt_handler(uint32_t id, uint32_t flags,
376 void *handle, void *cookie)
377{
378 int pri, idx, intr, intr_raw, ret = 0;
379 ehf_handler_t handler;
380
381 /*
382 * Top-level interrupt type handler from Interrupt Management Framework
383 * doesn't acknowledge the interrupt; so the interrupt ID must be
384 * invalid.
385 */
386 assert(id == INTR_ID_UNAVAILABLE);
387
388 /*
389 * Acknowledge interrupt. Proceed with handling only for valid interrupt
390 * IDs. This situation may arise because of Interrupt Management
391 * Framework identifying an EL3 interrupt, but before it's been
392 * acknowledged here, the interrupt was either deasserted, or there was
393 * a higher-priority interrupt of another type.
394 */
395 intr_raw = plat_ic_acknowledge_interrupt();
396 intr = plat_ic_get_interrupt_id(intr_raw);
397 if (intr == INTR_ID_UNAVAILABLE)
398 return 0;
399
400 /* Having acknowledged the interrupt, get the running priority */
401 pri = plat_ic_get_running_priority();
402
403 /* Check EL3 interrupt priority is in secure range */
404 assert(IS_PRI_SECURE(pri));
405
406 /*
407 * Translate the priority to a descriptor index. We do this by masking
408 * and shifting the running priority value (platform-supplied).
409 */
410 idx = pri_to_idx(pri);
411
412 /* Validate priority */
413 assert(pri == IDX_TO_PRI(idx));
414
415 handler = RAW_HANDLER(exception_data.ehf_priorities[idx].ehf_handler);
416 if (!handler) {
417 ERROR("No EL3 exception handler for priority 0x%x\n",
418 IDX_TO_PRI(idx));
419 panic();
420 }
421
422 /*
423 * Call registered handler. Pass the raw interrupt value to registered
424 * handlers.
425 */
426 ret = handler(intr_raw, flags, handle, cookie);
427
428 return ret;
429}
430
431/*
432 * Initialize the EL3 exception handling.
433 */
434void ehf_init(void)
435{
436 unsigned int flags = 0;
437 int ret __unused;
438
439 /* Ensure EL3 interrupts are supported */
440 assert(plat_ic_has_interrupt_type(INTR_TYPE_EL3));
441
442 /*
443 * Make sure that priority water mark has enough bits to represent the
444 * whole priority array.
445 */
446 assert(exception_data.num_priorities <= (sizeof(ehf_pri_bits_t) * 8));
447
448 assert(exception_data.ehf_priorities);
449
450 /*
451 * Bit 7 of GIC priority must be 0 for secure interrupts. This means
452 * platforms must use at least 1 of the remaining 7 bits.
453 */
454 assert((exception_data.pri_bits >= 1) || (exception_data.pri_bits < 8));
455
456 /* Route EL3 interrupts when in Secure and Non-secure. */
457 set_interrupt_rm_flag(flags, NON_SECURE);
458 set_interrupt_rm_flag(flags, SECURE);
459
460 /* Register handler for EL3 interrupts */
461 ret = register_interrupt_type_handler(INTR_TYPE_EL3,
462 ehf_el3_interrupt_handler, flags);
463 assert(ret == 0);
464}
465
466/*
467 * Register a handler at the supplied priority. Registration is allowed only if
468 * a handler hasn't been registered before, or one wasn't provided at build
469 * time. The priority for which the handler is being registered must also accord
470 * with the platform-supplied data.
471 */
472void ehf_register_priority_handler(unsigned int pri, ehf_handler_t handler)
473{
474 int idx;
475
476 /* Sanity check for handler */
477 assert(handler != NULL);
478
479 /* Handler ought to be 4-byte aligned */
480 assert((((uintptr_t) handler) & 3) == 0);
481
482 /* Ensure we register for valid priority */
483 idx = pri_to_idx(pri);
484 assert(idx < exception_data.num_priorities);
485 assert(IDX_TO_PRI(idx) == pri);
486
487 /* Return failure if a handler was already registered */
488 if (exception_data.ehf_priorities[idx].ehf_handler != _EHF_NO_HANDLER) {
489 ERROR("Handler already registered for priority 0x%x\n", pri);
490 panic();
491 }
492
493 /*
494 * Install handler, and retain the valid bit. We assume that the handler
495 * is 4-byte aligned, which is usually the case.
496 */
497 exception_data.ehf_priorities[idx].ehf_handler =
498 (((uintptr_t) handler) | _EHF_PRI_VALID);
499
500 EHF_LOG("register pri=0x%x handler=%p\n", pri, handler);
501}
Jeenu Viswambharan6c6f24d2017-10-04 12:21:34 +0100502
503SUBSCRIBE_TO_EVENT(cm_entering_normal_world, ehf_entering_normal_world);
504SUBSCRIBE_TO_EVENT(cm_exited_normal_world, ehf_exited_normal_world);