blob: d93e60d453bef8b99c5cd63cda000a7a12f948b3 [file] [log] [blame]
Soby Mathew991d42c2015-06-29 16:30:12 +01001/*
Zelalem Awekef92c0cb2022-01-31 16:59:42 -06002 * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved.
Soby Mathew991d42c2015-06-29 16:30:12 +01003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Soby Mathew991d42c2015-06-29 16:30:12 +01005 */
6
Antonio Nino Diaze0f90632018-12-14 00:18:21 +00007#include <assert.h>
8#include <stddef.h>
9
Soby Mathew991d42c2015-06-29 16:30:12 +010010#include <arch.h>
11#include <arch_helpers.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000012#include <common/bl_common.h>
13#include <common/debug.h>
Soby Mathew991d42c2015-06-29 16:30:12 +010014#include <context.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000015#include <lib/el3_runtime/context_mgmt.h>
16#include <lib/el3_runtime/cpu_data.h>
17#include <lib/el3_runtime/pubsub_events.h>
18#include <lib/pmf/pmf.h>
19#include <lib/runtime_instr.h>
20#include <plat/common/platform.h>
21
Soby Mathew991d42c2015-06-29 16:30:12 +010022#include "psci_private.h"
23
Soby Mathew991d42c2015-06-29 16:30:12 +010024/*******************************************************************************
Soby Mathew85dbf5a2015-04-07 12:16:56 +010025 * This function does generic and platform specific operations after a wake-up
26 * from standby/retention states at multiple power levels.
Soby Mathew991d42c2015-06-29 16:30:12 +010027 ******************************************************************************/
Deepika Bhavnani4287c0c2019-12-13 10:23:18 -060028static void psci_suspend_to_standby_finisher(unsigned int cpu_idx,
Soby Mathew85dbf5a2015-04-07 12:16:56 +010029 unsigned int end_pwrlvl)
Soby Mathew991d42c2015-06-29 16:30:12 +010030{
Andrew F. Davis74e89782019-06-04 10:46:54 -040031 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
Achin Gupta9b2bf252016-06-28 16:46:15 +010032 psci_power_state_t state_info;
33
Andrew F. Davis74e89782019-06-04 10:46:54 -040034 /* Get the parent nodes */
35 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes);
36
37 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes);
Soby Mathew991d42c2015-06-29 16:30:12 +010038
Soby Mathew85dbf5a2015-04-07 12:16:56 +010039 /*
Achin Gupta9b2bf252016-06-28 16:46:15 +010040 * Find out which retention states this CPU has exited from until the
41 * 'end_pwrlvl'. The exit retention state could be deeper than the entry
42 * state as a result of state coordination amongst other CPUs post wfi.
43 */
44 psci_get_target_local_pwr_states(end_pwrlvl, &state_info);
45
Soby Mathew8336f682017-10-16 15:19:31 +010046#if ENABLE_PSCI_STAT
47 plat_psci_stat_accounting_stop(&state_info);
48 psci_stats_update_pwr_up(end_pwrlvl, &state_info);
49#endif
50
Achin Gupta9b2bf252016-06-28 16:46:15 +010051 /*
Soby Mathew85dbf5a2015-04-07 12:16:56 +010052 * Plat. management: Allow the platform to do operations
53 * on waking up from retention.
54 */
Achin Gupta9b2bf252016-06-28 16:46:15 +010055 psci_plat_pm_ops->pwr_domain_suspend_finish(&state_info);
Soby Mathew991d42c2015-06-29 16:30:12 +010056
Soby Mathew85dbf5a2015-04-07 12:16:56 +010057 /*
58 * Set the requested and target state of this CPU and all the higher
59 * power domain levels for this CPU to run.
60 */
61 psci_set_pwr_domains_to_run(end_pwrlvl);
Soby Mathew991d42c2015-06-29 16:30:12 +010062
Andrew F. Davis74e89782019-06-04 10:46:54 -040063 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes);
Soby Mathew991d42c2015-06-29 16:30:12 +010064}
65
66/*******************************************************************************
Soby Mathew85dbf5a2015-04-07 12:16:56 +010067 * This function does generic and platform specific suspend to power down
68 * operations.
Soby Mathew991d42c2015-06-29 16:30:12 +010069 ******************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +010070static void psci_suspend_to_pwrdown_start(unsigned int end_pwrlvl,
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +010071 const entry_point_info_t *ep,
72 const psci_power_state_t *state_info)
Soby Mathew991d42c2015-06-29 16:30:12 +010073{
Achin Gupta9a0ff9b2015-09-07 20:43:27 +010074 unsigned int max_off_lvl = psci_find_max_off_lvl(state_info);
75
Dimitris Papastamosd1a18412017-11-28 15:16:00 +000076 PUBLISH_EVENT(psci_suspend_pwrdown_start);
77
Wing Li2c556f32022-09-14 13:18:17 -070078#if PSCI_OS_INIT_MODE
79#ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL
80 end_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL;
81#else
82 end_pwrlvl = PLAT_MAX_PWR_LVL;
83#endif
84#endif
85
Soby Mathew85dbf5a2015-04-07 12:16:56 +010086 /* Save PSCI target power level for the suspend finisher handler */
87 psci_set_suspend_pwrlvl(end_pwrlvl);
Soby Mathew991d42c2015-06-29 16:30:12 +010088
Soby Mathew85dbf5a2015-04-07 12:16:56 +010089 /*
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +000090 * Flush the target power level as it might be accessed on power up with
Soby Mathew85dbf5a2015-04-07 12:16:56 +010091 * Data cache disabled.
92 */
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +000093 psci_flush_cpu_data(psci_svc_cpu_data.target_pwrlvl);
Soby Mathew991d42c2015-06-29 16:30:12 +010094
Soby Mathew85dbf5a2015-04-07 12:16:56 +010095 /*
96 * Call the cpu suspend handler registered by the Secure Payload
97 * Dispatcher to let it do any book-keeping. If the handler encounters an
98 * error, it's expected to assert within
99 */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100100 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend != NULL))
Achin Gupta9a0ff9b2015-09-07 20:43:27 +0100101 psci_spd_pm->svc_suspend(max_off_lvl);
Soby Mathew991d42c2015-06-29 16:30:12 +0100102
Varun Wadekarae87f4b2017-07-10 16:02:05 -0700103#if !HW_ASSISTED_COHERENCY
104 /*
105 * Plat. management: Allow the platform to perform any early
106 * actions required to power down the CPU. This might be useful for
107 * HW_ASSISTED_COHERENCY = 0 platforms that can safely perform these
108 * actions with data caches enabled.
109 */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100110 if (psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early != NULL)
Varun Wadekarae87f4b2017-07-10 16:02:05 -0700111 psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early(state_info);
112#endif
113
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100114 /*
115 * Store the re-entry information for the non-secure world.
116 */
117 cm_init_my_context(ep);
Soby Mathew991d42c2015-06-29 16:30:12 +0100118
dp-arm2d92de62016-11-15 13:25:30 +0000119#if ENABLE_RUNTIME_INSTRUMENTATION
120
121 /*
122 * Flush cache line so that even if CPU power down happens
123 * the timestamp update is reflected in memory.
124 */
125 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
126 RT_INSTR_ENTER_CFLUSH,
127 PMF_CACHE_MAINT);
128#endif
129
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100130 /*
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +0000131 * Arch. management. Initiate power down sequence.
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100132 * TODO : Introduce a mechanism to query the cache level to flush
133 * and the cpu-ops power down to perform from the platform.
134 */
Pranav Madhuc1e61d02022-07-22 23:11:16 +0530135 psci_pwrdown_cpu(max_off_lvl);
dp-arm2d92de62016-11-15 13:25:30 +0000136
137#if ENABLE_RUNTIME_INSTRUMENTATION
138 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
139 RT_INSTR_EXIT_CFLUSH,
140 PMF_NO_CACHE_MAINT);
141#endif
Soby Mathew991d42c2015-06-29 16:30:12 +0100142}
143
144/*******************************************************************************
Soby Mathew991d42c2015-06-29 16:30:12 +0100145 * Top level handler which is called when a cpu wants to suspend its execution.
Soby Mathew3a9e8bf2015-05-05 16:33:16 +0100146 * It is assumed that along with suspending the cpu power domain, power domains
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100147 * at higher levels until the target power level will be suspended as well. It
148 * coordinates with the platform to negotiate the target state for each of
149 * the power domain level till the target power domain level. It then performs
150 * generic, architectural, platform setup and state management required to
151 * suspend that power domain level and power domain levels below it.
152 * e.g. For a cpu that's to be suspended, it could mean programming the
153 * power controller whereas for a cluster that's to be suspended, it will call
154 * the platform specific code which will disable coherency at the interconnect
155 * level if the cpu is the last in the cluster and also the program the power
156 * controller.
Soby Mathew991d42c2015-06-29 16:30:12 +0100157 *
158 * All the required parameter checks are performed at the beginning and after
Soby Mathew6b8b3022015-06-30 11:00:24 +0100159 * the state transition has been done, no further error is expected and it is
160 * not possible to undo any of the actions taken beyond that point.
Soby Mathew991d42c2015-06-29 16:30:12 +0100161 ******************************************************************************/
Wing Li2c556f32022-09-14 13:18:17 -0700162int psci_cpu_suspend_start(const entry_point_info_t *ep,
163 unsigned int end_pwrlvl,
164 psci_power_state_t *state_info,
165 unsigned int is_power_down_state)
Soby Mathew991d42c2015-06-29 16:30:12 +0100166{
Wing Li2c556f32022-09-14 13:18:17 -0700167 int rc = PSCI_E_SUCCESS;
168 bool skip_wfi = false;
Deepika Bhavnani4287c0c2019-12-13 10:23:18 -0600169 unsigned int idx = plat_my_core_pos();
Andrew F. Davis74e89782019-06-04 10:46:54 -0400170 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
Soby Mathew991d42c2015-06-29 16:30:12 +0100171
172 /*
173 * This function must only be called on platforms where the
174 * CPU_SUSPEND platform hooks have been implemented.
175 */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100176 assert((psci_plat_pm_ops->pwr_domain_suspend != NULL) &&
177 (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL));
Soby Mathew991d42c2015-06-29 16:30:12 +0100178
Andrew F. Davis74e89782019-06-04 10:46:54 -0400179 /* Get the parent nodes */
180 psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes);
181
Soby Mathew991d42c2015-06-29 16:30:12 +0100182 /*
Soby Mathew3a9e8bf2015-05-05 16:33:16 +0100183 * This function acquires the lock corresponding to each power
Soby Mathew991d42c2015-06-29 16:30:12 +0100184 * level so that by the time all locks are taken, the system topology
185 * is snapshot and state management can be done safely.
186 */
Andrew F. Davis74e89782019-06-04 10:46:54 -0400187 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes);
Soby Mathew991d42c2015-06-29 16:30:12 +0100188
189 /*
190 * We check if there are any pending interrupts after the delay
191 * introduced by lock contention to increase the chances of early
192 * detection that a wake-up interrupt has fired.
193 */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100194 if (read_isr_el1() != 0U) {
Wing Li2c556f32022-09-14 13:18:17 -0700195 skip_wfi = true;
Soby Mathew991d42c2015-06-29 16:30:12 +0100196 goto exit;
197 }
198
Wing Li2c556f32022-09-14 13:18:17 -0700199#if PSCI_OS_INIT_MODE
200 if (psci_suspend_mode == OS_INIT) {
201 /*
202 * This function validates the requested state info for
203 * OS-initiated mode.
204 */
205 rc = psci_validate_state_coordination(end_pwrlvl, state_info);
206 if (rc != PSCI_E_SUCCESS) {
207 skip_wfi = true;
208 goto exit;
209 }
210 } else {
211#endif
212 /*
213 * This function is passed the requested state info and
214 * it returns the negotiated state info for each power level upto
215 * the end level specified.
216 */
217 psci_do_state_coordination(end_pwrlvl, state_info);
218#if PSCI_OS_INIT_MODE
219 }
220#endif
Soby Mathew991d42c2015-06-29 16:30:12 +0100221
Wing Lic0dc6392023-05-04 08:31:19 -0700222#if PSCI_OS_INIT_MODE
223 if (psci_plat_pm_ops->pwr_domain_validate_suspend != NULL) {
224 rc = psci_plat_pm_ops->pwr_domain_validate_suspend(state_info);
225 if (rc != PSCI_E_SUCCESS) {
226 skip_wfi = true;
227 goto exit;
228 }
229 }
230#endif
231
232 /* Update the target state in the power domain nodes */
233 psci_set_target_local_pwr_states(end_pwrlvl, state_info);
234
Yatharth Kochar241ec6c2016-05-09 18:26:35 +0100235#if ENABLE_PSCI_STAT
236 /* Update the last cpu for each level till end_pwrlvl */
237 psci_stats_update_pwr_down(end_pwrlvl, state_info);
238#endif
239
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100240 if (is_power_down_state != 0U)
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100241 psci_suspend_to_pwrdown_start(end_pwrlvl, ep, state_info);
Soby Mathew991d42c2015-06-29 16:30:12 +0100242
Soby Mathew6b8b3022015-06-30 11:00:24 +0100243 /*
244 * Plat. management: Allow the platform to perform the
245 * necessary actions to turn off this cpu e.g. set the
246 * platform defined mailbox with the psci entrypoint,
247 * program the power controller etc.
248 */
Wing Li2c556f32022-09-14 13:18:17 -0700249
Sandrine Bailleux574d6852015-06-11 10:46:48 +0100250 psci_plat_pm_ops->pwr_domain_suspend(state_info);
Soby Mathew991d42c2015-06-29 16:30:12 +0100251
Yatharth Kochar241ec6c2016-05-09 18:26:35 +0100252#if ENABLE_PSCI_STAT
dp-arm66abfbe2017-01-31 13:01:04 +0000253 plat_psci_stat_accounting_start(state_info);
Yatharth Kochar241ec6c2016-05-09 18:26:35 +0100254#endif
255
Soby Mathew991d42c2015-06-29 16:30:12 +0100256exit:
257 /*
Soby Mathew3a9e8bf2015-05-05 16:33:16 +0100258 * Release the locks corresponding to each power level in the
Soby Mathew991d42c2015-06-29 16:30:12 +0100259 * reverse order to which they were acquired.
260 */
Andrew F. Davis74e89782019-06-04 10:46:54 -0400261 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes);
262
Wing Li2c556f32022-09-14 13:18:17 -0700263 if (skip_wfi) {
264 return rc;
265 }
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100266
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100267 if (is_power_down_state != 0U) {
dp-arm3cac7862016-09-19 11:18:44 +0100268#if ENABLE_RUNTIME_INSTRUMENTATION
269
270 /*
271 * Update the timestamp with cache off. We assume this
272 * timestamp can only be read from the current CPU and the
273 * timestamp cache line will be flushed before return to
274 * normal world on wakeup.
275 */
276 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
277 RT_INSTR_ENTER_HW_LOW_PWR,
278 PMF_NO_CACHE_MAINT);
279#endif
280
Soby Mathew6a816412016-04-27 14:46:28 +0100281 /* The function calls below must not return */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100282 if (psci_plat_pm_ops->pwr_domain_pwr_down_wfi != NULL)
Soby Mathew6a816412016-04-27 14:46:28 +0100283 psci_plat_pm_ops->pwr_domain_pwr_down_wfi(state_info);
284 else
285 psci_power_down_wfi();
286 }
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100287
dp-arm3cac7862016-09-19 11:18:44 +0100288#if ENABLE_RUNTIME_INSTRUMENTATION
289 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
290 RT_INSTR_ENTER_HW_LOW_PWR,
291 PMF_NO_CACHE_MAINT);
292#endif
293
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100294 /*
295 * We will reach here if only retention/standby states have been
296 * requested at multiple power levels. This means that the cpu
297 * context will be preserved.
298 */
299 wfi();
300
dp-arm3cac7862016-09-19 11:18:44 +0100301#if ENABLE_RUNTIME_INSTRUMENTATION
302 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
303 RT_INSTR_EXIT_HW_LOW_PWR,
304 PMF_NO_CACHE_MAINT);
305#endif
306
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100307 /*
308 * After we wake up from context retaining suspend, call the
309 * context retaining suspend finisher.
310 */
Achin Gupta9b2bf252016-06-28 16:46:15 +0100311 psci_suspend_to_standby_finisher(idx, end_pwrlvl);
Wing Li2c556f32022-09-14 13:18:17 -0700312
313 return rc;
Soby Mathew991d42c2015-06-29 16:30:12 +0100314}
315
316/*******************************************************************************
Soby Mathew3a9e8bf2015-05-05 16:33:16 +0100317 * The following functions finish an earlier suspend request. They
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100318 * are called by the common finisher routine in psci_common.c. The `state_info`
319 * is the psci_power_state from which this CPU has woken up from.
Soby Mathew991d42c2015-06-29 16:30:12 +0100320 ******************************************************************************/
Deepika Bhavnani4287c0c2019-12-13 10:23:18 -0600321void psci_cpu_suspend_finish(unsigned int cpu_idx, const psci_power_state_t *state_info)
Soby Mathew991d42c2015-06-29 16:30:12 +0100322{
Antonio Nino Diaz391a76e2016-05-18 16:53:31 +0100323 unsigned int counter_freq;
Achin Gupta9a0ff9b2015-09-07 20:43:27 +0100324 unsigned int max_off_lvl;
Soby Mathew991d42c2015-06-29 16:30:12 +0100325
Soby Mathew991d42c2015-06-29 16:30:12 +0100326 /* Ensure we have been woken up from a suspended state */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100327 assert((psci_get_aff_info_state() == AFF_STATE_ON) &&
328 (is_local_state_off(
329 state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]) != 0));
Soby Mathew991d42c2015-06-29 16:30:12 +0100330
331 /*
332 * Plat. management: Perform the platform specific actions
333 * before we change the state of the cpu e.g. enabling the
334 * gic or zeroing the mailbox register. If anything goes
335 * wrong then assert as there is no way to recover from this
336 * situation.
337 */
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100338 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info);
Soby Mathew991d42c2015-06-29 16:30:12 +0100339
Soby Mathew043fe9c2017-04-10 22:35:42 +0100340#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +0000341 /* Arch. management: Enable the data cache, stack memory maintenance. */
Soby Mathew991d42c2015-06-29 16:30:12 +0100342 psci_do_pwrup_cache_maintenance();
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +0000343#endif
Soby Mathew991d42c2015-06-29 16:30:12 +0100344
345 /* Re-init the cntfrq_el0 register */
Antonio Nino Diaz391a76e2016-05-18 16:53:31 +0100346 counter_freq = plat_get_syscnt_freq2();
Soby Mathew991d42c2015-06-29 16:30:12 +0100347 write_cntfrq_el0(counter_freq);
348
Alexei Fedorovf41355c2019-09-13 14:11:59 +0100349#if ENABLE_PAUTH
350 /* Store APIAKey_EL1 key */
351 set_cpu_data(apiakey[0], read_apiakeylo_el1());
352 set_cpu_data(apiakey[1], read_apiakeyhi_el1());
353#endif /* ENABLE_PAUTH */
354
Soby Mathew991d42c2015-06-29 16:30:12 +0100355 /*
356 * Call the cpu suspend finish handler registered by the Secure Payload
357 * Dispatcher to let it do any bookeeping. If the handler encounters an
358 * error, it's expected to assert within
359 */
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100360 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend_finish != NULL)) {
Achin Gupta9a0ff9b2015-09-07 20:43:27 +0100361 max_off_lvl = psci_find_max_off_lvl(state_info);
Antonio Nino Diaz56a0e8e2018-07-16 23:19:25 +0100362 assert(max_off_lvl != PSCI_INVALID_PWR_LVL);
Achin Gupta9a0ff9b2015-09-07 20:43:27 +0100363 psci_spd_pm->svc_suspend_finish(max_off_lvl);
Soby Mathew991d42c2015-06-29 16:30:12 +0100364 }
365
Soby Mathew85dbf5a2015-04-07 12:16:56 +0100366 /* Invalidate the suspend level for the cpu */
Soby Mathew011ca182015-07-29 17:05:03 +0100367 psci_set_suspend_pwrlvl(PSCI_INVALID_PWR_LVL);
Soby Mathew991d42c2015-06-29 16:30:12 +0100368
Dimitris Papastamosd1a18412017-11-28 15:16:00 +0000369 PUBLISH_EVENT(psci_suspend_pwrdown_finish);
370
Soby Mathew991d42c2015-06-29 16:30:12 +0100371 /*
372 * Generic management: Now we just need to retrieve the
373 * information that we had stashed away during the suspend
374 * call to set this cpu on its way.
375 */
Zelalem Awekef92c0cb2022-01-31 16:59:42 -0600376 cm_prepare_el3_exit_ns();
Soby Mathew991d42c2015-06-29 16:30:12 +0100377}