blob: 42bf46d0f92e21fb205e9e8405c7615034b6433a [file] [log] [blame]
Jeenu Viswambharan04e3a7f2017-10-16 08:43:14 +01001/*
2 * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <arch_helpers.h>
8#include <assert.h>
9#include <bl_common.h>
10#include <cassert.h>
11#include <context_mgmt.h>
12#include <debug.h>
13#include <ehf.h>
14#include <interrupt_mgmt.h>
15#include <runtime_svc.h>
16#include <sdei.h>
17#include <string.h>
18#include "sdei_private.h"
19
20#define PE_MASKED 1
21#define PE_NOT_MASKED 0
22
23/* x0-x17 GPREGS context */
24#define SDEI_SAVED_GPREGS 18
25
26/* Maximum preemption nesting levels: Critical priority and Normal priority */
27#define MAX_EVENT_NESTING 2
28
29/* Per-CPU SDEI state access macro */
30#define sdei_get_this_pe_state() (&sdei_cpu_state[plat_my_core_pos()])
31
32/* Structure to store information about an outstanding dispatch */
33typedef struct sdei_dispatch_context {
34 sdei_ev_map_t *map;
35 unsigned int sec_state;
36 unsigned int intr_raw;
37 uint64_t x[SDEI_SAVED_GPREGS];
38
39 /* Exception state registers */
40 uint64_t elr_el3;
41 uint64_t spsr_el3;
42} sdei_dispatch_context_t;
43
44/* Per-CPU SDEI state data */
45typedef struct sdei_cpu_state {
46 sdei_dispatch_context_t dispatch_stack[MAX_EVENT_NESTING];
47 unsigned short stack_top; /* Empty ascending */
48 unsigned int pe_masked:1;
49 unsigned int pending_enables:1;
50} sdei_cpu_state_t;
51
52/* SDEI states for all cores in the system */
53static sdei_cpu_state_t sdei_cpu_state[PLATFORM_CORE_COUNT];
54
55unsigned int sdei_pe_mask(void)
56{
57 unsigned int ret;
58 sdei_cpu_state_t *state = sdei_get_this_pe_state();
59
60 /*
61 * Return value indicates whether this call had any effect in the mask
62 * status of this PE.
63 */
64 ret = (state->pe_masked ^ PE_MASKED);
65 state->pe_masked = PE_MASKED;
66
67 return ret;
68}
69
70void sdei_pe_unmask(void)
71{
72 int i;
73 sdei_ev_map_t *map;
74 sdei_entry_t *se;
75 sdei_cpu_state_t *state = sdei_get_this_pe_state();
76 uint64_t my_mpidr = read_mpidr_el1() & MPIDR_AFFINITY_MASK;
77
78 /*
79 * If there are pending enables, iterate through the private mappings
80 * and enable those bound maps that are in enabled state. Also, iterate
81 * through shared mappings and enable interrupts of events that are
82 * targeted to this PE.
83 */
84 if (state->pending_enables) {
85 for_each_private_map(i, map) {
86 se = get_event_entry(map);
87 if (is_map_bound(map) && GET_EV_STATE(se, ENABLED))
88 plat_ic_enable_interrupt(map->intr);
89 }
90
91 for_each_shared_map(i, map) {
92 se = get_event_entry(map);
93
94 sdei_map_lock(map);
95 if (is_map_bound(map) &&
96 GET_EV_STATE(se, ENABLED) &&
97 (se->reg_flags == SDEI_REGF_RM_PE) &&
98 (se->affinity == my_mpidr)) {
99 plat_ic_enable_interrupt(map->intr);
100 }
101 sdei_map_unlock(map);
102 }
103 }
104
105 state->pending_enables = 0;
106 state->pe_masked = PE_NOT_MASKED;
107}
108
109/* Push a dispatch context to the dispatch stack */
110static sdei_dispatch_context_t *push_dispatch(void)
111{
112 sdei_cpu_state_t *state = sdei_get_this_pe_state();
113 sdei_dispatch_context_t *disp_ctx;
114
115 /* Cannot have more than max events */
116 assert(state->stack_top < MAX_EVENT_NESTING);
117
118 disp_ctx = &state->dispatch_stack[state->stack_top];
119 state->stack_top++;
120
121 return disp_ctx;
122}
123
124/* Pop a dispatch context to the dispatch stack */
125static sdei_dispatch_context_t *pop_dispatch(void)
126{
127 sdei_cpu_state_t *state = sdei_get_this_pe_state();
128
129 if (state->stack_top == 0)
130 return NULL;
131
132 assert(state->stack_top <= MAX_EVENT_NESTING);
133
134 state->stack_top--;
135
136 return &state->dispatch_stack[state->stack_top];
137}
138
139/* Retrieve the context at the top of dispatch stack */
140static sdei_dispatch_context_t *get_outstanding_dispatch(void)
141{
142 sdei_cpu_state_t *state = sdei_get_this_pe_state();
143
144 if (state->stack_top == 0)
145 return NULL;
146
147 assert(state->stack_top <= MAX_EVENT_NESTING);
148
149 return &state->dispatch_stack[state->stack_top - 1];
150}
151
152static void save_event_ctx(sdei_ev_map_t *map, void *tgt_ctx, int sec_state,
153 unsigned int intr_raw)
154{
155 sdei_dispatch_context_t *disp_ctx;
156 gp_regs_t *tgt_gpregs;
157 el3_state_t *tgt_el3;
158
159 assert(tgt_ctx);
160 tgt_gpregs = get_gpregs_ctx(tgt_ctx);
161 tgt_el3 = get_el3state_ctx(tgt_ctx);
162
163 disp_ctx = push_dispatch();
164 assert(disp_ctx);
165 disp_ctx->sec_state = sec_state;
166 disp_ctx->map = map;
167 disp_ctx->intr_raw = intr_raw;
168
169 /* Save general purpose and exception registers */
170 memcpy(disp_ctx->x, tgt_gpregs, sizeof(disp_ctx->x));
171 disp_ctx->spsr_el3 = read_ctx_reg(tgt_el3, CTX_SPSR_EL3);
172 disp_ctx->elr_el3 = read_ctx_reg(tgt_el3, CTX_ELR_EL3);
173}
174
175static void restore_event_ctx(sdei_dispatch_context_t *disp_ctx, void *tgt_ctx)
176{
177 gp_regs_t *tgt_gpregs;
178 el3_state_t *tgt_el3;
179
180 assert(tgt_ctx);
181 tgt_gpregs = get_gpregs_ctx(tgt_ctx);
182 tgt_el3 = get_el3state_ctx(tgt_ctx);
183
184 CASSERT(sizeof(disp_ctx->x) == (SDEI_SAVED_GPREGS * sizeof(uint64_t)),
185 foo);
186
187 /* Restore general purpose and exception registers */
188 memcpy(tgt_gpregs, disp_ctx->x, sizeof(disp_ctx->x));
189 write_ctx_reg(tgt_el3, CTX_SPSR_EL3, disp_ctx->spsr_el3);
190 write_ctx_reg(tgt_el3, CTX_ELR_EL3, disp_ctx->elr_el3);
191}
192
193static void save_secure_context(void)
194{
195 cm_el1_sysregs_context_save(SECURE);
196}
197
198/* Restore Secure context and arrange to resume it at the next ERET */
199static void restore_and_resume_secure_context(void)
200{
201 cm_el1_sysregs_context_restore(SECURE);
202 cm_set_next_eret_context(SECURE);
203}
204
205/*
206 * Restore Non-secure context and arrange to resume it at the next ERET. Return
207 * pointer to the Non-secure context.
208 */
209static cpu_context_t *restore_and_resume_ns_context(void)
210{
211 cpu_context_t *ns_ctx;
212
213 cm_el1_sysregs_context_restore(NON_SECURE);
214 cm_set_next_eret_context(NON_SECURE);
215
216 ns_ctx = cm_get_context(NON_SECURE);
217 assert(ns_ctx);
218
219 return ns_ctx;
220}
221
222/*
223 * Populate the Non-secure context so that the next ERET will dispatch to the
224 * SDEI client.
225 */
226static void setup_ns_dispatch(sdei_ev_map_t *map, sdei_entry_t *se,
227 cpu_context_t *ctx, int sec_state_to_resume,
228 unsigned int intr_raw)
229{
230 el3_state_t *el3_ctx = get_el3state_ctx(ctx);
231
232 /* Push the event and context */
233 save_event_ctx(map, ctx, sec_state_to_resume, intr_raw);
234
235 /*
236 * Setup handler arguments:
237 *
238 * - x0: Event number
239 * - x1: Handler argument supplied at the time of event registration
240 * - x2: Interrupted PC
241 * - x3: Interrupted SPSR
242 */
243 SMC_SET_GP(ctx, CTX_GPREG_X0, map->ev_num);
244 SMC_SET_GP(ctx, CTX_GPREG_X1, se->arg);
245 SMC_SET_GP(ctx, CTX_GPREG_X2, read_ctx_reg(el3_ctx, CTX_ELR_EL3));
246 SMC_SET_GP(ctx, CTX_GPREG_X3, read_ctx_reg(el3_ctx, CTX_SPSR_EL3));
247
248 /*
249 * Prepare for ERET:
250 *
251 * - Set PC to the registered handler address
252 * - Set SPSR to jump to client EL with exceptions masked
253 */
254 cm_set_elr_spsr_el3(NON_SECURE, (uintptr_t) se->ep,
255 SPSR_64(sdei_client_el(), MODE_SP_ELX,
256 DISABLE_ALL_EXCEPTIONS));
257}
258
259/* Handle a triggered SDEI interrupt while events were masked on this PE */
260static void handle_masked_trigger(sdei_ev_map_t *map, sdei_entry_t *se,
261 sdei_cpu_state_t *state, unsigned int intr_raw)
262{
263 uint64_t my_mpidr __unused = (read_mpidr_el1() & MPIDR_AFFINITY_MASK);
264 int disable = 0;
265
266 /* Nothing to do for event 0 */
267 if (map->ev_num == SDEI_EVENT_0)
268 return;
269
270 /*
271 * For a private event, or for a shared event specifically routed to
272 * this CPU, we disable interrupt, leave the interrupt pending, and do
273 * EOI.
274 */
275 if (is_event_private(map)) {
276 disable = 1;
277 } else if (se->reg_flags == SDEI_REGF_RM_PE) {
278 assert(se->affinity == my_mpidr);
279 disable = 1;
280 }
281
282 if (disable) {
283 plat_ic_disable_interrupt(map->intr);
284 plat_ic_set_interrupt_pending(map->intr);
285 plat_ic_end_of_interrupt(intr_raw);
286 state->pending_enables = 1;
287
288 return;
289 }
290
291 /*
292 * We just received a shared event with routing set to ANY PE. The
293 * interrupt can't be delegated on this PE as SDEI events are masked.
294 * However, because its routing mode is ANY, it is possible that the
295 * event can be delegated on any other PE that hasn't masked events.
296 * Therefore, we set the interrupt back pending so as to give other
297 * suitable PEs a chance of handling it.
298 */
299 assert(plat_ic_is_spi(map->intr));
300 plat_ic_set_interrupt_pending(map->intr);
301
302 /*
303 * Leaving the same interrupt pending also means that the same interrupt
304 * can target this PE again as soon as this PE leaves EL3. Whether and
305 * how often that happens depends on the implementation of GIC.
306 *
307 * We therefore call a platform handler to resolve this situation.
308 */
309 plat_sdei_handle_masked_trigger(my_mpidr, map->intr);
310
311 /* This PE is masked. We EOI the interrupt, as it can't be delegated */
312 plat_ic_end_of_interrupt(intr_raw);
313}
314
315/* SDEI main interrupt handler */
316int sdei_intr_handler(uint32_t intr_raw, uint32_t flags, void *handle,
317 void *cookie)
318{
319 sdei_entry_t *se;
320 cpu_context_t *ctx;
321 sdei_ev_map_t *map;
322 sdei_dispatch_context_t *disp_ctx;
323 unsigned int sec_state;
324 sdei_cpu_state_t *state;
325 uint32_t intr;
326
327 /*
328 * To handle an event, the following conditions must be true:
329 *
330 * 1. Event must be signalled
331 * 2. Event must be enabled
332 * 3. This PE must be a target PE for the event
333 * 4. PE must be unmasked for SDEI
334 * 5. If this is a normal event, no event must be running
335 * 6. If this is a critical event, no critical event must be running
336 *
337 * (1) and (2) are true when this function is running
338 * (3) is enforced in GIC by selecting the appropriate routing option
339 * (4) is satisfied by client calling PE_UNMASK
340 * (5) and (6) is enforced using interrupt priority, the RPR, in GIC:
341 * - Normal SDEI events belong to Normal SDE priority class
342 * - Critical SDEI events belong to Critical CSDE priority class
343 *
344 * The interrupt has already been acknowledged, and therefore is active,
345 * so no other PE can handle this event while we are at it.
346 *
347 * Find if this is an SDEI interrupt. There must be an event mapped to
348 * this interrupt
349 */
350 intr = plat_ic_get_interrupt_id(intr_raw);
351 map = find_event_map_by_intr(intr, plat_ic_is_spi(intr));
352 if (!map) {
353 ERROR("No SDEI map for interrupt %u\n", intr);
354 panic();
355 }
356
357 /*
358 * Received interrupt number must either correspond to event 0, or must
359 * be bound interrupt.
360 */
361 assert((map->ev_num == SDEI_EVENT_0) || is_map_bound(map));
362
363 se = get_event_entry(map);
364 state = sdei_get_this_pe_state();
365
366 if (state->pe_masked == PE_MASKED) {
367 /*
368 * Interrupts received while this PE was masked can't be
369 * dispatched.
370 */
371 SDEI_LOG("interrupt %u on %lx while PE masked\n", map->intr,
372 read_mpidr_el1());
373 if (is_event_shared(map))
374 sdei_map_lock(map);
375
376 handle_masked_trigger(map, se, state, intr_raw);
377
378 if (is_event_shared(map))
379 sdei_map_unlock(map);
380
381 return 0;
382 }
383
384 /* Insert load barrier for signalled SDEI event */
385 if (map->ev_num == SDEI_EVENT_0)
386 dmbld();
387
388 if (is_event_shared(map))
389 sdei_map_lock(map);
390
391 /* Assert shared event routed to this PE had been configured so */
392 if (is_event_shared(map) && (se->reg_flags == SDEI_REGF_RM_PE)) {
393 assert(se->affinity ==
394 (read_mpidr_el1() & MPIDR_AFFINITY_MASK));
395 }
396
397 if (!can_sdei_state_trans(se, DO_DISPATCH)) {
398 SDEI_LOG("SDEI event 0x%x can't be dispatched; state=0x%x\n",
399 map->ev_num, se->state);
400
401 /*
402 * If the event is registered, leave the interrupt pending so
403 * that it's delivered when the event is enabled.
404 */
405 if (GET_EV_STATE(se, REGISTERED))
406 plat_ic_set_interrupt_pending(map->intr);
407
408 /*
409 * The interrupt was disabled or unregistered after the handler
410 * started to execute, which means now the interrupt is already
411 * disabled and we just need to EOI the interrupt.
412 */
413 plat_ic_end_of_interrupt(intr_raw);
414
415 if (is_event_shared(map))
416 sdei_map_unlock(map);
417
418 return 0;
419 }
420
421 disp_ctx = get_outstanding_dispatch();
422 if (is_event_critical(map)) {
423 /*
424 * If this event is Critical, and if there's an outstanding
425 * dispatch, assert the latter is a Normal dispatch. Critical
426 * events can preempt an outstanding Normal event dispatch.
427 */
428 if (disp_ctx)
429 assert(is_event_normal(disp_ctx->map));
430 } else {
431 /*
432 * If this event is Normal, assert that there are no outstanding
433 * dispatches. Normal events can't preempt any outstanding event
434 * dispatches.
435 */
436 assert(disp_ctx == NULL);
437 }
438
439 sec_state = get_interrupt_src_ss(flags);
440
441 if (is_event_shared(map))
442 sdei_map_unlock(map);
443
444 SDEI_LOG("ACK %lx, ev:%d ss:%d spsr:%lx ELR:%lx\n", read_mpidr_el1(),
445 map->ev_num, sec_state, read_spsr_el3(),
446 read_elr_el3());
447
448 ctx = handle;
449
450 /*
451 * Check if we interrupted secure state. Perform a context switch so
452 * that we can delegate to NS.
453 */
454 if (sec_state == SECURE) {
455 save_secure_context();
456 ctx = restore_and_resume_ns_context();
457 }
458
459 setup_ns_dispatch(map, se, ctx, sec_state, intr_raw);
460
461 /*
462 * End of interrupt is done in sdei_event_complete, when the client
463 * signals completion.
464 */
465 return 0;
466}
467
Jeenu Viswambharancf1f2212017-10-02 12:10:54 +0100468/* Explicitly dispatch the given SDEI event */
469int sdei_dispatch_event(int ev_num, unsigned int preempted_sec_state)
470{
471 sdei_entry_t *se;
472 sdei_ev_map_t *map;
473 cpu_context_t *ctx;
474 sdei_dispatch_context_t *disp_ctx;
475 sdei_cpu_state_t *state;
476
477 /* Validate preempted security state */
Jeenu Viswambharan5b58e1a2017-11-14 10:52:20 +0000478 if ((preempted_sec_state != SECURE) &&
479 (preempted_sec_state != NON_SECURE)) {
Jeenu Viswambharancf1f2212017-10-02 12:10:54 +0100480 return -1;
Jeenu Viswambharan5b58e1a2017-11-14 10:52:20 +0000481 }
Jeenu Viswambharancf1f2212017-10-02 12:10:54 +0100482
483 /* Can't dispatch if events are masked on this PE */
484 state = sdei_get_this_pe_state();
485 if (state->pe_masked == PE_MASKED)
486 return -1;
487
488 /* Event 0 can't be dispatched */
489 if (ev_num == SDEI_EVENT_0)
490 return -1;
491
492 /* Locate mapping corresponding to this event */
493 map = find_event_map(ev_num);
494 if (!map)
495 return -1;
496
497 /*
498 * Statically-bound or dynamic maps are dispatched only as a result of
499 * interrupt, and not upon explicit request.
500 */
501 if (is_map_dynamic(map) || is_map_bound(map))
502 return -1;
503
504 /* The event must be private */
505 if (is_event_shared(map))
506 return -1;
507
508 /* Examine state of dispatch stack */
509 disp_ctx = get_outstanding_dispatch();
510 if (disp_ctx) {
511 /*
512 * There's an outstanding dispatch. If the outstanding dispatch
513 * is critical, no more dispatches are possible.
514 */
515 if (is_event_critical(disp_ctx->map))
516 return -1;
517
518 /*
519 * If the outstanding dispatch is Normal, only critical events
520 * can be dispatched.
521 */
522 if (is_event_normal(map))
523 return -1;
524 }
525
526 se = get_event_entry(map);
527 if (!can_sdei_state_trans(se, DO_DISPATCH))
528 return -1;
529
530 /* Activate the priority corresponding to the event being dispatched */
531 ehf_activate_priority(sdei_event_priority(map));
532
533 /*
534 * We assume the current context is SECURE, and that it's already been
535 * saved.
536 */
537 ctx = restore_and_resume_ns_context();
538
539 /*
540 * The caller has effectively terminated execution. Record to resume the
541 * preempted context later when the event completes or
542 * complete-and-resumes.
543 */
544 setup_ns_dispatch(map, se, ctx, preempted_sec_state, 0);
545
546 return 0;
547}
548
Jeenu Viswambharan04e3a7f2017-10-16 08:43:14 +0100549int sdei_event_complete(int resume, uint64_t pc)
550{
551 sdei_dispatch_context_t *disp_ctx;
552 sdei_entry_t *se;
553 sdei_ev_map_t *map;
554 cpu_context_t *ctx;
555 sdei_action_t act;
556 unsigned int client_el = sdei_client_el();
557
558 /* Return error if called without an active event */
559 disp_ctx = pop_dispatch();
560 if (!disp_ctx)
561 return SDEI_EDENY;
562
563 /* Validate resumption point */
564 if (resume && (plat_sdei_validate_entry_point(pc, client_el) != 0))
565 return SDEI_EDENY;
566
567 map = disp_ctx->map;
568 assert(map);
569
570 se = get_event_entry(map);
571
572 SDEI_LOG("EOI:%lx, %d spsr:%lx elr:%lx\n", read_mpidr_el1(),
573 map->ev_num, read_spsr_el3(), read_elr_el3());
574
575 if (is_event_shared(map))
576 sdei_map_lock(map);
577
578 act = resume ? DO_COMPLETE_RESUME : DO_COMPLETE;
579 if (!can_sdei_state_trans(se, act)) {
580 if (is_event_shared(map))
581 sdei_map_unlock(map);
582 return SDEI_EDENY;
583 }
584
585 /*
586 * Restore Non-secure to how it was originally interrupted. Once done,
587 * it's up-to-date with the saved copy.
588 */
589 ctx = cm_get_context(NON_SECURE);
590 restore_event_ctx(disp_ctx, ctx);
591
592 if (resume) {
593 /*
594 * Complete-and-resume call. Prepare the Non-secure context
595 * (currently active) for complete and resume.
596 */
597 cm_set_elr_spsr_el3(NON_SECURE, pc, SPSR_64(client_el,
598 MODE_SP_ELX, DISABLE_ALL_EXCEPTIONS));
599
600 /*
601 * Make it look as if a synchronous exception were taken at the
602 * supplied Non-secure resumption point. Populate SPSR and
603 * ELR_ELx so that an ERET from there works as expected.
604 *
605 * The assumption is that the client, if necessary, would have
606 * saved any live content in these registers before making this
607 * call.
608 */
609 if (client_el == MODE_EL2) {
610 write_elr_el2(disp_ctx->elr_el3);
611 write_spsr_el2(disp_ctx->spsr_el3);
612 } else {
613 /* EL1 */
614 write_elr_el1(disp_ctx->elr_el3);
615 write_spsr_el1(disp_ctx->spsr_el3);
616 }
617 }
618
619 /*
620 * If the cause of dispatch originally interrupted the Secure world, and
621 * if Non-secure world wasn't allowed to preempt Secure execution,
622 * resume Secure.
623 *
624 * No need to save the Non-secure context ahead of a world switch: the
625 * Non-secure context was fully saved before dispatch, and has been
626 * returned to its pre-dispatch state.
627 */
628 if ((disp_ctx->sec_state == SECURE) &&
629 (ehf_is_ns_preemption_allowed() == 0)) {
630 restore_and_resume_secure_context();
631 }
632
633 if ((map->ev_num == SDEI_EVENT_0) || is_map_bound(map)) {
634 /*
635 * The event was dispatched after receiving SDEI interrupt. With
636 * the event handling completed, EOI the corresponding
637 * interrupt.
638 */
639 plat_ic_end_of_interrupt(disp_ctx->intr_raw);
Jeenu Viswambharancf1f2212017-10-02 12:10:54 +0100640 } else {
641 /*
642 * An unbound event must have been dispatched explicitly.
643 * Deactivate the priority level that was activated at the time
644 * of explicit dispatch.
645 */
646 ehf_deactivate_priority(sdei_event_priority(map));
Jeenu Viswambharan04e3a7f2017-10-16 08:43:14 +0100647 }
648
649 if (is_event_shared(map))
650 sdei_map_unlock(map);
651
652 return 0;
653}
654
655int sdei_event_context(void *handle, unsigned int param)
656{
657 sdei_dispatch_context_t *disp_ctx;
658
659 if (param >= SDEI_SAVED_GPREGS)
660 return SDEI_EINVAL;
661
662 /* Get outstanding dispatch on this CPU */
663 disp_ctx = get_outstanding_dispatch();
664 if (!disp_ctx)
665 return SDEI_EDENY;
666
667 assert(disp_ctx->map);
668
669 if (!can_sdei_state_trans(get_event_entry(disp_ctx->map), DO_CONTEXT))
670 return SDEI_EDENY;
671
672 /*
673 * No locking is required for the Running status as this is the only CPU
674 * which can complete the event
675 */
676
677 return disp_ctx->x[param];
678}