blob: 8aa0cce717001b4090f7e3862f0b06deace8a173 [file] [log] [blame]
Achin Gupta4f6ad662013-10-25 09:08:21 +01001/*
Yann Gautier507e0cd2022-02-14 11:09:23 +01002 * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved.
Achin Gupta4f6ad662013-10-25 09:08:21 +01003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Achin Gupta4f6ad662013-10-25 09:08:21 +01005 */
6
Antonio Nino Diaze0f90632018-12-14 00:18:21 +00007#include <assert.h>
8#include <string.h>
9
Dan Handley2bd4ef22014-04-09 13:14:54 +010010#include <arch.h>
Achin Gupta4f6ad662013-10-25 09:08:21 +010011#include <arch_helpers.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000012#include <common/bl_common.h>
13#include <common/debug.h>
Dan Handley2bd4ef22014-04-09 13:14:54 +010014#include <context.h>
Sandeep Tripathy12030042020-08-17 20:22:13 +053015#include <drivers/delay_timer.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000016#include <lib/el3_runtime/context_mgmt.h>
17#include <lib/utils.h>
18#include <plat/common/platform.h>
19
Dan Handley714a0d22014-04-09 13:13:04 +010020#include "psci_private.h"
Achin Gupta4f6ad662013-10-25 09:08:21 +010021
Achin Gupta607084e2014-02-09 18:24:19 +000022/*
Jeenu Viswambharan7f366602014-02-20 17:11:00 +000023 * SPD power management operations, expected to be supplied by the registered
24 * SPD on successful SP initialization
Achin Gupta607084e2014-02-09 18:24:19 +000025 */
Dan Handleye2712bc2014-04-10 15:37:22 +010026const spd_pm_ops_t *psci_spd_pm;
Achin Gupta607084e2014-02-09 18:24:19 +000027
Soby Mathew981487a2015-07-13 14:10:57 +010028/*
29 * PSCI requested local power state map. This array is used to store the local
30 * power states requested by a CPU for power levels from level 1 to
31 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power
32 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a
33 * CPU are the same.
34 *
35 * During state coordination, the platform is passed an array containing the
36 * local states requested for a particular non cpu power domain by each cpu
37 * within the domain.
38 *
39 * TODO: Dense packing of the requested states will cause cache thrashing
40 * when multiple power domains write to it. If we allocate the requested
41 * states at each power level in a cache-line aligned per-domain memory,
42 * the cache thrashing can be avoided.
43 */
44static plat_local_state_t
45 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT];
46
Pankaj Gupta02c35682019-10-15 15:44:45 +053047unsigned int psci_plat_core_count;
Soby Mathew981487a2015-07-13 14:10:57 +010048
Achin Gupta4f6ad662013-10-25 09:08:21 +010049/*******************************************************************************
Soby Mathew981487a2015-07-13 14:10:57 +010050 * Arrays that hold the platform's power domain tree information for state
51 * management of power domains.
52 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain
53 * which is an ancestor of a CPU power domain.
54 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain
Achin Gupta4f6ad662013-10-25 09:08:21 +010055 ******************************************************************************/
Soby Mathew981487a2015-07-13 14:10:57 +010056non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS]
Soby Mathew2ae20432015-01-08 18:02:44 +000057#if USE_COHERENT_MEM
Chris Kay33bfc5e2023-02-14 11:30:04 +000058__section(".tzfw_coherent_mem")
Soby Mathew2ae20432015-01-08 18:02:44 +000059#endif
60;
Achin Gupta4f6ad662013-10-25 09:08:21 +010061
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +000062/* Lock for PSCI state coordination */
63DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]);
Andrew Thoelkee466c9f2015-09-10 11:39:36 +010064
Soby Mathew981487a2015-07-13 14:10:57 +010065cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT];
66
Achin Gupta4f6ad662013-10-25 09:08:21 +010067/*******************************************************************************
Achin Gupta4f6ad662013-10-25 09:08:21 +010068 * Pointer to functions exported by the platform to complete power mgmt. ops
69 ******************************************************************************/
Soby Mathew981487a2015-07-13 14:10:57 +010070const plat_psci_ops_t *psci_plat_pm_ops;
Achin Gupta4f6ad662013-10-25 09:08:21 +010071
Soby Mathew981487a2015-07-13 14:10:57 +010072/******************************************************************************
73 * Check that the maximum power level supported by the platform makes sense
74 *****************************************************************************/
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +010075CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) &&
76 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL),
77 assert_platform_max_pwrlvl_check);
Soby Mathew2b7de2b2015-02-12 14:45:02 +000078
Wing Li71f69df2022-09-14 13:18:15 -070079#if PSCI_OS_INIT_MODE
80/*******************************************************************************
81 * The power state coordination mode used in CPU_SUSPEND.
82 * Defaults to platform-coordinated mode.
83 ******************************************************************************/
84suspend_mode_t psci_suspend_mode = PLAT_COORD;
85#endif
86
Soby Mathew981487a2015-07-13 14:10:57 +010087/*
88 * The plat_local_state used by the platform is one of these types: RUN,
89 * RETENTION and OFF. The platform can define further sub-states for each type
90 * apart from RUN. This categorization is done to verify the sanity of the
91 * psci_power_state passed by the platform and to print debug information. The
92 * categorization is done on the basis of the following conditions:
93 *
94 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN.
95 *
96 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is
97 * STATE_TYPE_RETN.
98 *
99 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is
100 * STATE_TYPE_OFF.
101 */
102typedef enum plat_local_state_type {
103 STATE_TYPE_RUN = 0,
104 STATE_TYPE_RETN,
105 STATE_TYPE_OFF
106} plat_local_state_type_t;
107
Antonio Nino Diaz5a42b682018-07-18 11:57:21 +0100108/* Function used to categorize plat_local_state. */
109static plat_local_state_type_t find_local_state_type(plat_local_state_t state)
110{
111 if (state != 0U) {
112 if (state > PLAT_MAX_RET_STATE) {
113 return STATE_TYPE_OFF;
114 } else {
115 return STATE_TYPE_RETN;
116 }
117 } else {
118 return STATE_TYPE_RUN;
119 }
120}
Soby Mathew981487a2015-07-13 14:10:57 +0100121
122/******************************************************************************
123 * Check that the maximum retention level supported by the platform is less
124 * than the maximum off level.
125 *****************************************************************************/
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100126CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE,
Soby Mathew981487a2015-07-13 14:10:57 +0100127 assert_platform_max_off_and_retn_state_check);
128
129/******************************************************************************
130 * This function ensures that the power state parameter in a CPU_SUSPEND request
131 * is valid. If so, it returns the requested states for each power level.
132 *****************************************************************************/
133int psci_validate_power_state(unsigned int power_state,
134 psci_power_state_t *state_info)
Achin Guptaf6b9e992014-07-31 11:19:11 +0100135{
Soby Mathew981487a2015-07-13 14:10:57 +0100136 /* Check SBZ bits in power state are zero */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100137 if (psci_check_power_state(power_state) != 0U)
Soby Mathew981487a2015-07-13 14:10:57 +0100138 return PSCI_E_INVALID_PARAMS;
Achin Guptaf6b9e992014-07-31 11:19:11 +0100139
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100140 assert(psci_plat_pm_ops->validate_power_state != NULL);
Achin Guptaf6b9e992014-07-31 11:19:11 +0100141
Soby Mathew981487a2015-07-13 14:10:57 +0100142 /* Validate the power_state using platform pm_ops */
143 return psci_plat_pm_ops->validate_power_state(power_state, state_info);
144}
Achin Guptaf6b9e992014-07-31 11:19:11 +0100145
Soby Mathew981487a2015-07-13 14:10:57 +0100146/******************************************************************************
147 * This function retrieves the `psci_power_state_t` for system suspend from
148 * the platform.
149 *****************************************************************************/
150void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info)
151{
152 /*
153 * Assert that the required pm_ops hook is implemented to ensure that
154 * the capability detected during psci_setup() is valid.
155 */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100156 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL);
Soby Mathew981487a2015-07-13 14:10:57 +0100157
158 /*
159 * Query the platform for the power_state required for system suspend
160 */
161 psci_plat_pm_ops->get_sys_suspend_power_state(state_info);
Achin Guptaf6b9e992014-07-31 11:19:11 +0100162}
163
Wing Li2c556f32022-09-14 13:18:17 -0700164#if PSCI_OS_INIT_MODE
165/*******************************************************************************
166 * This function verifies that all the other cores at the 'end_pwrlvl' have been
167 * idled and the current CPU is the last running CPU at the 'end_pwrlvl'.
168 * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false)
169 * otherwise.
170 ******************************************************************************/
171static bool psci_is_last_cpu_to_idle_at_pwrlvl(unsigned int end_pwrlvl)
172{
173 unsigned int my_idx, lvl, parent_idx;
174 unsigned int cpu_start_idx, ncpus, cpu_idx;
175 plat_local_state_t local_state;
176
177 if (end_pwrlvl == PSCI_CPU_PWR_LVL) {
178 return true;
179 }
180
181 my_idx = plat_my_core_pos();
182
183 for (lvl = PSCI_CPU_PWR_LVL; lvl <= end_pwrlvl; lvl++) {
184 parent_idx = psci_cpu_pd_nodes[my_idx].parent_node;
185 }
186
187 cpu_start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx;
188 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus;
189
190 for (cpu_idx = cpu_start_idx; cpu_idx < cpu_start_idx + ncpus;
191 cpu_idx++) {
192 local_state = psci_get_cpu_local_state_by_idx(cpu_idx);
193 if (cpu_idx == my_idx) {
194 assert(is_local_state_run(local_state) != 0);
195 continue;
196 }
197
198 if (is_local_state_run(local_state) != 0) {
199 return false;
200 }
201 }
202
203 return true;
204}
205#endif
206
Achin Guptaf6b9e992014-07-31 11:19:11 +0100207/*******************************************************************************
Wing Li71f69df2022-09-14 13:18:15 -0700208 * This function verifies that all the other cores in the system have been
Soby Mathew96168382014-12-17 14:47:57 +0000209 * turned OFF and the current CPU is the last running CPU in the system.
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +0100210 * Returns true, if the current CPU is the last ON CPU or false otherwise.
Soby Mathew96168382014-12-17 14:47:57 +0000211 ******************************************************************************/
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +0100212bool psci_is_last_on_cpu(void)
Soby Mathew96168382014-12-17 14:47:57 +0000213{
Deepika Bhavnani79ffab52019-08-27 00:32:24 +0300214 unsigned int cpu_idx, my_idx = plat_my_core_pos();
Soby Mathew96168382014-12-17 14:47:57 +0000215
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +0100216 for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) {
Soby Mathew981487a2015-07-13 14:10:57 +0100217 if (cpu_idx == my_idx) {
218 assert(psci_get_aff_info_state() == AFF_STATE_ON);
Soby Mathew96168382014-12-17 14:47:57 +0000219 continue;
220 }
221
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +0100222 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) {
223 VERBOSE("core=%u other than current core=%u %s\n",
224 cpu_idx, my_idx, "running in the system");
225 return false;
226 }
Soby Mathew96168382014-12-17 14:47:57 +0000227 }
228
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +0100229 return true;
Soby Mathew96168382014-12-17 14:47:57 +0000230}
231
232/*******************************************************************************
Wing Li71f69df2022-09-14 13:18:15 -0700233 * This function verifies that all cores in the system have been turned ON.
234 * Returns true, if all CPUs are ON or false otherwise.
235 ******************************************************************************/
236static bool psci_are_all_cpus_on(void)
237{
238 unsigned int cpu_idx;
239
240 for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) {
241 if (psci_get_aff_info_state_by_idx(cpu_idx) == AFF_STATE_OFF) {
242 return false;
243 }
244 }
245
246 return true;
247}
248
249/*******************************************************************************
Soby Mathew981487a2015-07-13 14:10:57 +0100250 * Routine to return the maximum power level to traverse to after a cpu has
251 * been physically powered up. It is expected to be called immediately after
252 * reset from assembler code.
Achin Guptaf6b9e992014-07-31 11:19:11 +0100253 ******************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +0100254static unsigned int get_power_on_target_pwrlvl(void)
Achin Guptaf6b9e992014-07-31 11:19:11 +0100255{
Soby Mathew011ca182015-07-29 17:05:03 +0100256 unsigned int pwrlvl;
Achin Guptaf6b9e992014-07-31 11:19:11 +0100257
258 /*
Soby Mathew981487a2015-07-13 14:10:57 +0100259 * Assume that this cpu was suspended and retrieve its target power
260 * level. If it is invalid then it could only have been turned off
261 * earlier. PLAT_MAX_PWR_LVL will be the highest power level a
262 * cpu can be turned off to.
Achin Guptaf6b9e992014-07-31 11:19:11 +0100263 */
Soby Mathew981487a2015-07-13 14:10:57 +0100264 pwrlvl = psci_get_suspend_pwrlvl();
Soby Mathew011ca182015-07-29 17:05:03 +0100265 if (pwrlvl == PSCI_INVALID_PWR_LVL)
Soby Mathew981487a2015-07-13 14:10:57 +0100266 pwrlvl = PLAT_MAX_PWR_LVL;
Deepika Bhavnani523024c2019-08-17 01:10:02 +0300267 assert(pwrlvl < PSCI_INVALID_PWR_LVL);
Soby Mathew981487a2015-07-13 14:10:57 +0100268 return pwrlvl;
Achin Guptaf6b9e992014-07-31 11:19:11 +0100269}
270
Soby Mathew981487a2015-07-13 14:10:57 +0100271/******************************************************************************
272 * Helper function to update the requested local power state array. This array
273 * does not store the requested state for the CPU power level. Hence an
Deepika Bhavnani6bd46662019-08-15 00:56:46 +0300274 * assertion is added to prevent us from accessing the CPU power level.
Soby Mathew981487a2015-07-13 14:10:57 +0100275 *****************************************************************************/
276static void psci_set_req_local_pwr_state(unsigned int pwrlvl,
277 unsigned int cpu_idx,
278 plat_local_state_t req_pwr_state)
Achin Guptaf6b9e992014-07-31 11:19:11 +0100279{
Soby Mathew981487a2015-07-13 14:10:57 +0100280 assert(pwrlvl > PSCI_CPU_PWR_LVL);
Deepika Bhavnani6bd46662019-08-15 00:56:46 +0300281 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) &&
Pankaj Gupta02c35682019-10-15 15:44:45 +0530282 (cpu_idx < psci_plat_core_count)) {
Deepika Bhavnani6bd46662019-08-15 00:56:46 +0300283 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state;
284 }
Achin Guptaf6b9e992014-07-31 11:19:11 +0100285}
286
Soby Mathew981487a2015-07-13 14:10:57 +0100287/******************************************************************************
288 * This function initializes the psci_req_local_pwr_states.
289 *****************************************************************************/
Daniel Boulby5753e492018-09-20 14:12:46 +0100290void __init psci_init_req_local_pwr_states(void)
Achin Guptaa45e3972013-12-05 15:10:48 +0000291{
Soby Mathew981487a2015-07-13 14:10:57 +0100292 /* Initialize the requested state of all non CPU power domains as OFF */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100293 unsigned int pwrlvl;
Pankaj Gupta02c35682019-10-15 15:44:45 +0530294 unsigned int core;
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100295
296 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) {
Pankaj Gupta02c35682019-10-15 15:44:45 +0530297 for (core = 0; core < psci_plat_core_count; core++) {
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100298 psci_req_local_pwr_states[pwrlvl][core] =
299 PLAT_MAX_OFF_STATE;
300 }
301 }
Soby Mathew981487a2015-07-13 14:10:57 +0100302}
Achin Guptaa45e3972013-12-05 15:10:48 +0000303
Soby Mathew981487a2015-07-13 14:10:57 +0100304/******************************************************************************
305 * Helper function to return a reference to an array containing the local power
306 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the
307 * array will be the number of cpu power domains of which this power domain is
308 * an ancestor. These requested states will be used to determine a suitable
309 * target state for this power domain during psci state coordination. An
310 * assertion is added to prevent us from accessing the CPU power level.
311 *****************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +0100312static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl,
Deepika Bhavnani79ffab52019-08-27 00:32:24 +0300313 unsigned int cpu_idx)
Soby Mathew981487a2015-07-13 14:10:57 +0100314{
315 assert(pwrlvl > PSCI_CPU_PWR_LVL);
Achin Guptaf3ccbab2014-07-25 14:52:47 +0100316
Deepika Bhavnani6bd46662019-08-15 00:56:46 +0300317 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) &&
Pankaj Gupta02c35682019-10-15 15:44:45 +0530318 (cpu_idx < psci_plat_core_count)) {
Deepika Bhavnani6bd46662019-08-15 00:56:46 +0300319 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx];
320 } else
321 return NULL;
Soby Mathew981487a2015-07-13 14:10:57 +0100322}
Achin Guptaa45e3972013-12-05 15:10:48 +0000323
Wing Li2c556f32022-09-14 13:18:17 -0700324#if PSCI_OS_INIT_MODE
325/******************************************************************************
326 * Helper function to save a copy of the psci_req_local_pwr_states (prev) for a
327 * CPU (cpu_idx), and update psci_req_local_pwr_states with the new requested
328 * local power states (state_info).
329 *****************************************************************************/
330void psci_update_req_local_pwr_states(unsigned int end_pwrlvl,
331 unsigned int cpu_idx,
332 psci_power_state_t *state_info,
333 plat_local_state_t *prev)
334{
335 unsigned int lvl;
336#ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL
337 unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL;
338#else
339 unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL;
340#endif
341 plat_local_state_t req_state;
342
343 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) {
344 /* Save the previous requested local power state */
345 prev[lvl - 1U] = *psci_get_req_local_pwr_states(lvl, cpu_idx);
346
347 /* Update the new requested local power state */
348 if (lvl <= end_pwrlvl) {
349 req_state = state_info->pwr_domain_state[lvl];
350 } else {
351 req_state = state_info->pwr_domain_state[end_pwrlvl];
352 }
353 psci_set_req_local_pwr_state(lvl, cpu_idx, req_state);
354 }
355}
356
357/******************************************************************************
358 * Helper function to restore the previously saved requested local power states
359 * (prev) for a CPU (cpu_idx) to psci_req_local_pwr_states.
360 *****************************************************************************/
361void psci_restore_req_local_pwr_states(unsigned int cpu_idx,
362 plat_local_state_t *prev)
363{
364 unsigned int lvl;
365#ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL
366 unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL;
367#else
368 unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL;
369#endif
370
371 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) {
372 /* Restore the previous requested local power state */
373 psci_set_req_local_pwr_state(lvl, cpu_idx, prev[lvl - 1U]);
374 }
375}
376#endif
377
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000378/*
379 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent
380 * memory.
381 *
382 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory,
383 * it's accessed by both cached and non-cached participants. To serve the common
384 * minimum, perform a cache flush before read and after write so that non-cached
385 * participants operate on latest data in main memory.
386 *
387 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent
388 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent.
389 * In both cases, no cache operations are required.
390 */
391
392/*
393 * Retrieve local state of non-CPU power domain node from a non-cached CPU,
394 * after any required cache maintenance operation.
395 */
396static plat_local_state_t get_non_cpu_pd_node_local_state(
397 unsigned int parent_idx)
398{
Andrew F. Davise6f28fa2018-08-30 12:13:57 -0500399#if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000400 flush_dcache_range(
401 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx],
402 sizeof(psci_non_cpu_pd_nodes[parent_idx]));
403#endif
404 return psci_non_cpu_pd_nodes[parent_idx].local_state;
405}
406
407/*
408 * Update local state of non-CPU power domain node from a cached CPU; perform
409 * any required cache maintenance operation afterwards.
410 */
411static void set_non_cpu_pd_node_local_state(unsigned int parent_idx,
412 plat_local_state_t state)
413{
414 psci_non_cpu_pd_nodes[parent_idx].local_state = state;
Andrew F. Davise6f28fa2018-08-30 12:13:57 -0500415#if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000416 flush_dcache_range(
417 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx],
418 sizeof(psci_non_cpu_pd_nodes[parent_idx]));
419#endif
420}
421
Soby Mathew981487a2015-07-13 14:10:57 +0100422/******************************************************************************
423 * Helper function to return the current local power state of each power domain
424 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This
425 * function will be called after a cpu is powered on to find the local state
426 * each power domain has emerged from.
427 *****************************************************************************/
Achin Gupta9b2bf252016-06-28 16:46:15 +0100428void psci_get_target_local_pwr_states(unsigned int end_pwrlvl,
429 psci_power_state_t *target_state)
Soby Mathew981487a2015-07-13 14:10:57 +0100430{
Soby Mathew011ca182015-07-29 17:05:03 +0100431 unsigned int parent_idx, lvl;
Soby Mathew981487a2015-07-13 14:10:57 +0100432 plat_local_state_t *pd_state = target_state->pwr_domain_state;
433
434 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state();
435 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
436
437 /* Copy the local power state from node to state_info */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100438 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000439 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx);
Soby Mathew981487a2015-07-13 14:10:57 +0100440 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
441 }
442
443 /* Set the the higher levels to RUN */
444 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++)
445 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN;
446}
447
448/******************************************************************************
449 * Helper function to set the target local power state that each power domain
450 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will
451 * enter. This function will be called after coordination of requested power
452 * states has been done for each power level.
453 *****************************************************************************/
Wing Lic0dc6392023-05-04 08:31:19 -0700454void psci_set_target_local_pwr_states(unsigned int end_pwrlvl,
455 const psci_power_state_t *target_state)
Soby Mathew981487a2015-07-13 14:10:57 +0100456{
Soby Mathew011ca182015-07-29 17:05:03 +0100457 unsigned int parent_idx, lvl;
Soby Mathew981487a2015-07-13 14:10:57 +0100458 const plat_local_state_t *pd_state = target_state->pwr_domain_state;
459
460 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]);
Achin Guptaa45e3972013-12-05 15:10:48 +0000461
Achin Guptaf3ccbab2014-07-25 14:52:47 +0100462 /*
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000463 * Need to flush as local_state might be accessed with Data Cache
Soby Mathew981487a2015-07-13 14:10:57 +0100464 * disabled during power on
Achin Guptaf3ccbab2014-07-25 14:52:47 +0100465 */
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000466 psci_flush_cpu_data(psci_svc_cpu_data.local_state);
Soby Mathew981487a2015-07-13 14:10:57 +0100467
468 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
469
470 /* Copy the local_state from state_info */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100471 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) {
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000472 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]);
Soby Mathew981487a2015-07-13 14:10:57 +0100473 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
474 }
Achin Guptaa45e3972013-12-05 15:10:48 +0000475}
476
477/*******************************************************************************
Soby Mathew981487a2015-07-13 14:10:57 +0100478 * PSCI helper function to get the parent nodes corresponding to a cpu_index.
Achin Gupta4f6ad662013-10-25 09:08:21 +0100479 ******************************************************************************/
Deepika Bhavnani79ffab52019-08-27 00:32:24 +0300480void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx,
Soby Mathew011ca182015-07-29 17:05:03 +0100481 unsigned int end_lvl,
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100482 unsigned int *node_index)
Soby Mathew981487a2015-07-13 14:10:57 +0100483{
484 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node;
Varun Wadekar66231d12017-06-07 09:57:42 -0700485 unsigned int i;
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100486 unsigned int *node = node_index;
Soby Mathew981487a2015-07-13 14:10:57 +0100487
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100488 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) {
489 *node = parent_node;
490 node++;
Soby Mathew981487a2015-07-13 14:10:57 +0100491 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node;
492 }
493}
494
495/******************************************************************************
496 * This function is invoked post CPU power up and initialization. It sets the
497 * affinity info state, target power state and requested power state for the
498 * current CPU and all its ancestor power domains to RUN.
499 *****************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +0100500void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl)
Soby Mathew981487a2015-07-13 14:10:57 +0100501{
Soby Mathew011ca182015-07-29 17:05:03 +0100502 unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl;
Soby Mathew981487a2015-07-13 14:10:57 +0100503 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
504
505 /* Reset the local_state to RUN for the non cpu power domains. */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100506 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000507 set_non_cpu_pd_node_local_state(parent_idx,
508 PSCI_LOCAL_STATE_RUN);
Soby Mathew981487a2015-07-13 14:10:57 +0100509 psci_set_req_local_pwr_state(lvl,
510 cpu_idx,
511 PSCI_LOCAL_STATE_RUN);
512 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
513 }
514
515 /* Set the affinity info state to ON */
516 psci_set_aff_info_state(AFF_STATE_ON);
517
518 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN);
Jeenu Viswambharan0b56d6f2017-01-06 14:58:11 +0000519 psci_flush_cpu_data(psci_svc_cpu_data);
Soby Mathew981487a2015-07-13 14:10:57 +0100520}
521
522/******************************************************************************
Wing Li2c556f32022-09-14 13:18:17 -0700523 * This function is used in platform-coordinated mode.
524 *
Soby Mathew981487a2015-07-13 14:10:57 +0100525 * This function is passed the local power states requested for each power
526 * domain (state_info) between the current CPU domain and its ancestors until
527 * the target power level (end_pwrlvl). It updates the array of requested power
528 * states with this information.
529 *
530 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it
531 * retrieves the states requested by all the cpus of which the power domain at
532 * that level is an ancestor. It passes this information to the platform to
533 * coordinate and return the target power state. If the target state for a level
534 * is RUN then subsequent levels are not considered. At the CPU level, state
535 * coordination is not required. Hence, the requested and the target states are
536 * the same.
537 *
538 * The 'state_info' is updated with the target state for each level between the
539 * CPU and the 'end_pwrlvl' and returned to the caller.
540 *
541 * This function will only be invoked with data cache enabled and while
542 * powering down a core.
543 *****************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +0100544void psci_do_state_coordination(unsigned int end_pwrlvl,
545 psci_power_state_t *state_info)
Achin Gupta4f6ad662013-10-25 09:08:21 +0100546{
Soby Mathew981487a2015-07-13 14:10:57 +0100547 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos();
Deepika Bhavnani79ffab52019-08-27 00:32:24 +0300548 unsigned int start_idx;
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100549 unsigned int ncpus;
Soby Mathew981487a2015-07-13 14:10:57 +0100550 plat_local_state_t target_state, *req_states;
551
Soby Mathew1298e692016-02-02 14:23:10 +0000552 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL);
Soby Mathew981487a2015-07-13 14:10:57 +0100553 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
554
555 /* For level 0, the requested state will be equivalent
556 to target state */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100557 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
Soby Mathew981487a2015-07-13 14:10:57 +0100558
559 /* First update the requested power state */
560 psci_set_req_local_pwr_state(lvl, cpu_idx,
561 state_info->pwr_domain_state[lvl]);
562
563 /* Get the requested power states for this power level */
564 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx;
565 req_states = psci_get_req_local_pwr_states(lvl, start_idx);
566
567 /*
568 * Let the platform coordinate amongst the requested states at
569 * this power level and return the target local power state.
570 */
571 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus;
572 target_state = plat_get_target_pwr_state(lvl,
573 req_states,
574 ncpus);
Achin Gupta4f6ad662013-10-25 09:08:21 +0100575
Soby Mathew981487a2015-07-13 14:10:57 +0100576 state_info->pwr_domain_state[lvl] = target_state;
577
578 /* Break early if the negotiated target power state is RUN */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100579 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0)
Soby Mathew981487a2015-07-13 14:10:57 +0100580 break;
581
582 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
583 }
Achin Gupta4f6ad662013-10-25 09:08:21 +0100584
585 /*
Soby Mathew981487a2015-07-13 14:10:57 +0100586 * This is for cases when we break out of the above loop early because
587 * the target power state is RUN at a power level < end_pwlvl.
588 * We update the requested power state from state_info and then
589 * set the target state as RUN.
Achin Gupta4f6ad662013-10-25 09:08:21 +0100590 */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100591 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) {
Soby Mathew981487a2015-07-13 14:10:57 +0100592 psci_set_req_local_pwr_state(lvl, cpu_idx,
593 state_info->pwr_domain_state[lvl]);
594 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100595
Wing Li2c556f32022-09-14 13:18:17 -0700596 }
Wing Li2c556f32022-09-14 13:18:17 -0700597}
598
599#if PSCI_OS_INIT_MODE
600/******************************************************************************
601 * This function is used in OS-initiated mode.
602 *
603 * This function is passed the local power states requested for each power
604 * domain (state_info) between the current CPU domain and its ancestors until
605 * the target power level (end_pwrlvl), and ensures the requested power states
606 * are valid. It updates the array of requested power states with this
607 * information.
608 *
609 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it
610 * retrieves the states requested by all the cpus of which the power domain at
611 * that level is an ancestor. It passes this information to the platform to
612 * coordinate and return the target power state. If the requested state does
613 * not match the target state, the request is denied.
614 *
615 * The 'state_info' is not modified.
616 *
617 * This function will only be invoked with data cache enabled and while
618 * powering down a core.
619 *****************************************************************************/
620int psci_validate_state_coordination(unsigned int end_pwrlvl,
621 psci_power_state_t *state_info)
622{
623 int rc = PSCI_E_SUCCESS;
624 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos();
625 unsigned int start_idx;
626 unsigned int ncpus;
627 plat_local_state_t target_state, *req_states;
628 plat_local_state_t prev[PLAT_MAX_PWR_LVL];
629
630 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL);
631 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
632
633 /*
634 * Save a copy of the previous requested local power states and update
635 * the new requested local power states.
636 */
637 psci_update_req_local_pwr_states(end_pwrlvl, cpu_idx, state_info, prev);
638
639 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
640 /* Get the requested power states for this power level */
641 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx;
642 req_states = psci_get_req_local_pwr_states(lvl, start_idx);
643
644 /*
645 * Let the platform coordinate amongst the requested states at
646 * this power level and return the target local power state.
647 */
648 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus;
649 target_state = plat_get_target_pwr_state(lvl,
650 req_states,
651 ncpus);
652
653 /*
654 * Verify that the requested power state matches the target
655 * local power state.
656 */
657 if (state_info->pwr_domain_state[lvl] != target_state) {
658 if (target_state == PSCI_LOCAL_STATE_RUN) {
659 rc = PSCI_E_DENIED;
660 } else {
661 rc = PSCI_E_INVALID_PARAMS;
662 }
663 goto exit;
664 }
665 }
666
667 /*
668 * Verify that the current core is the last running core at the
669 * specified power level.
670 */
671 lvl = state_info->last_at_pwrlvl;
672 if (!psci_is_last_cpu_to_idle_at_pwrlvl(lvl)) {
673 rc = PSCI_E_DENIED;
674 }
675
676exit:
677 if (rc != PSCI_E_SUCCESS) {
678 /* Restore the previous requested local power states. */
679 psci_restore_req_local_pwr_states(cpu_idx, prev);
680 return rc;
Soby Mathew981487a2015-07-13 14:10:57 +0100681 }
Achin Gupta4f6ad662013-10-25 09:08:21 +0100682
Wing Li2c556f32022-09-14 13:18:17 -0700683 return rc;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100684}
Wing Li2c556f32022-09-14 13:18:17 -0700685#endif
Achin Gupta4f6ad662013-10-25 09:08:21 +0100686
Soby Mathew981487a2015-07-13 14:10:57 +0100687/******************************************************************************
688 * This function validates a suspend request by making sure that if a standby
689 * state is requested then no power level is turned off and the highest power
690 * level is placed in a standby/retention state.
691 *
692 * It also ensures that the state level X will enter is not shallower than the
693 * state level X + 1 will enter.
694 *
695 * This validation will be enabled only for DEBUG builds as the platform is
696 * expected to perform these validations as well.
697 *****************************************************************************/
698int psci_validate_suspend_req(const psci_power_state_t *state_info,
699 unsigned int is_power_down_state)
Achin Gupta0959db52013-12-02 17:33:04 +0000700{
Soby Mathew981487a2015-07-13 14:10:57 +0100701 unsigned int max_off_lvl, target_lvl, max_retn_lvl;
702 plat_local_state_t state;
703 plat_local_state_type_t req_state_type, deepest_state_type;
704 int i;
Achin Gupta0959db52013-12-02 17:33:04 +0000705
Soby Mathew981487a2015-07-13 14:10:57 +0100706 /* Find the target suspend power level */
707 target_lvl = psci_find_target_suspend_lvl(state_info);
Soby Mathew011ca182015-07-29 17:05:03 +0100708 if (target_lvl == PSCI_INVALID_PWR_LVL)
Achin Gupta0959db52013-12-02 17:33:04 +0000709 return PSCI_E_INVALID_PARAMS;
710
Soby Mathew981487a2015-07-13 14:10:57 +0100711 /* All power domain levels are in a RUN state to begin with */
712 deepest_state_type = STATE_TYPE_RUN;
713
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100714 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) {
Soby Mathew981487a2015-07-13 14:10:57 +0100715 state = state_info->pwr_domain_state[i];
716 req_state_type = find_local_state_type(state);
717
718 /*
719 * While traversing from the highest power level to the lowest,
720 * the state requested for lower levels has to be the same or
721 * deeper i.e. equal to or greater than the state at the higher
722 * levels. If this condition is true, then the requested state
723 * becomes the deepest state encountered so far.
724 */
725 if (req_state_type < deepest_state_type)
726 return PSCI_E_INVALID_PARAMS;
727 deepest_state_type = req_state_type;
728 }
729
730 /* Find the highest off power level */
731 max_off_lvl = psci_find_max_off_lvl(state_info);
732
733 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */
Soby Mathew011ca182015-07-29 17:05:03 +0100734 max_retn_lvl = PSCI_INVALID_PWR_LVL;
Soby Mathew981487a2015-07-13 14:10:57 +0100735 if (target_lvl != max_off_lvl)
736 max_retn_lvl = target_lvl;
737
738 /*
739 * If this is not a request for a power down state then max off level
740 * has to be invalid and max retention level has to be a valid power
741 * level.
742 */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100743 if ((is_power_down_state == 0U) &&
744 ((max_off_lvl != PSCI_INVALID_PWR_LVL) ||
745 (max_retn_lvl == PSCI_INVALID_PWR_LVL)))
Achin Gupta0959db52013-12-02 17:33:04 +0000746 return PSCI_E_INVALID_PARAMS;
747
748 return PSCI_E_SUCCESS;
749}
750
Soby Mathew981487a2015-07-13 14:10:57 +0100751/******************************************************************************
752 * This function finds the highest power level which will be powered down
753 * amongst all the power levels specified in the 'state_info' structure
754 *****************************************************************************/
755unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info)
Achin Guptacab78e42014-07-28 00:09:01 +0100756{
Soby Mathew981487a2015-07-13 14:10:57 +0100757 int i;
Achin Guptacab78e42014-07-28 00:09:01 +0100758
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100759 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) {
760 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0)
761 return (unsigned int) i;
Soby Mathew981487a2015-07-13 14:10:57 +0100762 }
763
Soby Mathew011ca182015-07-29 17:05:03 +0100764 return PSCI_INVALID_PWR_LVL;
Soby Mathew981487a2015-07-13 14:10:57 +0100765}
766
767/******************************************************************************
768 * This functions finds the level of the highest power domain which will be
769 * placed in a low power state during a suspend operation.
770 *****************************************************************************/
771unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info)
772{
773 int i;
774
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100775 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) {
776 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0)
777 return (unsigned int) i;
Achin Guptacab78e42014-07-28 00:09:01 +0100778 }
Soby Mathew981487a2015-07-13 14:10:57 +0100779
Soby Mathew011ca182015-07-29 17:05:03 +0100780 return PSCI_INVALID_PWR_LVL;
Achin Guptacab78e42014-07-28 00:09:01 +0100781}
782
783/*******************************************************************************
Andrew F. Davis74e89782019-06-04 10:46:54 -0400784 * This function is passed the highest level in the topology tree that the
785 * operation should be applied to and a list of node indexes. It picks up locks
786 * from the node index list in order of increasing power domain level in the
787 * range specified.
Achin Gupta0959db52013-12-02 17:33:04 +0000788 ******************************************************************************/
Andrew F. Davis74e89782019-06-04 10:46:54 -0400789void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl,
790 const unsigned int *parent_nodes)
Achin Gupta0959db52013-12-02 17:33:04 +0000791{
Andrew F. Davis74e89782019-06-04 10:46:54 -0400792 unsigned int parent_idx;
Soby Mathew011ca182015-07-29 17:05:03 +0100793 unsigned int level;
Achin Gupta0959db52013-12-02 17:33:04 +0000794
Soby Mathew981487a2015-07-13 14:10:57 +0100795 /* No locking required for level 0. Hence start locking from level 1 */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100796 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) {
Andrew F. Davis74e89782019-06-04 10:46:54 -0400797 parent_idx = parent_nodes[level - 1U];
Soby Mathew981487a2015-07-13 14:10:57 +0100798 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]);
Achin Gupta0959db52013-12-02 17:33:04 +0000799 }
800}
801
802/*******************************************************************************
Andrew F. Davis74e89782019-06-04 10:46:54 -0400803 * This function is passed the highest level in the topology tree that the
804 * operation should be applied to and a list of node indexes. It releases the
805 * locks in order of decreasing power domain level in the range specified.
Achin Gupta0959db52013-12-02 17:33:04 +0000806 ******************************************************************************/
Andrew F. Davis74e89782019-06-04 10:46:54 -0400807void psci_release_pwr_domain_locks(unsigned int end_pwrlvl,
808 const unsigned int *parent_nodes)
Achin Gupta0959db52013-12-02 17:33:04 +0000809{
Andrew F. Davis74e89782019-06-04 10:46:54 -0400810 unsigned int parent_idx;
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100811 unsigned int level;
Achin Gupta0959db52013-12-02 17:33:04 +0000812
Soby Mathew981487a2015-07-13 14:10:57 +0100813 /* Unlock top down. No unlocking required for level 0. */
Zelalem91d80612020-02-12 10:37:03 -0600814 for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) {
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100815 parent_idx = parent_nodes[level - 1U];
Soby Mathew981487a2015-07-13 14:10:57 +0100816 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]);
Achin Gupta0959db52013-12-02 17:33:04 +0000817 }
818}
819
820/*******************************************************************************
Soby Mathew981487a2015-07-13 14:10:57 +0100821 * Simple routine to determine whether a mpidr is valid or not.
Achin Gupta4f6ad662013-10-25 09:08:21 +0100822 ******************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +0100823int psci_validate_mpidr(u_register_t mpidr)
Achin Gupta4f6ad662013-10-25 09:08:21 +0100824{
Andre Przywara98c0b9a2023-04-27 13:46:41 +0100825 int pos = plat_core_pos_by_mpidr(mpidr);
826
827 if ((pos < 0) || ((unsigned int)pos >= PLATFORM_CORE_COUNT)) {
Achin Gupta4f6ad662013-10-25 09:08:21 +0100828 return PSCI_E_INVALID_PARAMS;
Andre Przywara98c0b9a2023-04-27 13:46:41 +0100829 }
Soby Mathew981487a2015-07-13 14:10:57 +0100830
831 return PSCI_E_SUCCESS;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100832}
833
834/*******************************************************************************
Andrew Thoelke4e126072014-06-04 21:10:52 +0100835 * This function determines the full entrypoint information for the requested
Soby Mathew8595b872015-01-06 15:36:38 +0000836 * PSCI entrypoint on power on/resume and returns it.
Achin Gupta4f6ad662013-10-25 09:08:21 +0100837 ******************************************************************************/
Julius Werner8e0ef0f2019-07-09 14:02:43 -0700838#ifdef __aarch64__
Soby Mathewf1f97a12015-07-15 12:13:26 +0100839static int psci_get_ns_ep_info(entry_point_info_t *ep,
Soby Mathew011ca182015-07-29 17:05:03 +0100840 uintptr_t entrypoint,
841 u_register_t context_id)
Achin Gupta4f6ad662013-10-25 09:08:21 +0100842{
Soby Mathewa0fedc42016-06-16 14:52:04 +0100843 u_register_t ep_attr, sctlr;
Soby Mathew011ca182015-07-29 17:05:03 +0100844 unsigned int daif, ee, mode;
Soby Mathewa0fedc42016-06-16 14:52:04 +0100845 u_register_t ns_scr_el3 = read_scr_el3();
846 u_register_t ns_sctlr_el1 = read_sctlr_el1();
Achin Gupta4f6ad662013-10-25 09:08:21 +0100847
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100848 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ?
849 read_sctlr_el2() : ns_sctlr_el1;
Andrew Thoelke4e126072014-06-04 21:10:52 +0100850 ee = 0;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100851
Andrew Thoelke4e126072014-06-04 21:10:52 +0100852 ep_attr = NON_SECURE | EP_ST_DISABLE;
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100853 if ((sctlr & SCTLR_EE_BIT) != 0U) {
Andrew Thoelke4e126072014-06-04 21:10:52 +0100854 ep_attr |= EP_EE_BIG;
855 ee = 1;
856 }
Soby Mathew8595b872015-01-06 15:36:38 +0000857 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr);
Achin Gupta4f6ad662013-10-25 09:08:21 +0100858
Soby Mathew8595b872015-01-06 15:36:38 +0000859 ep->pc = entrypoint;
Douglas Raillarda8954fc2017-01-26 15:54:44 +0000860 zeromem(&ep->args, sizeof(ep->args));
Soby Mathew8595b872015-01-06 15:36:38 +0000861 ep->args.arg0 = context_id;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100862
863 /*
864 * Figure out whether the cpu enters the non-secure address space
865 * in aarch32 or aarch64
866 */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100867 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) {
Achin Gupta4f6ad662013-10-25 09:08:21 +0100868
869 /*
870 * Check whether a Thumb entry point has been provided for an
871 * aarch64 EL
872 */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100873 if ((entrypoint & 0x1UL) != 0UL)
Soby Mathewf1f97a12015-07-15 12:13:26 +0100874 return PSCI_E_INVALID_ADDRESS;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100875
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100876 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100877
Jimmy Brissoned202072020-08-04 16:18:52 -0500878 ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX,
879 DISABLE_ALL_EXCEPTIONS);
Achin Gupta4f6ad662013-10-25 09:08:21 +0100880 } else {
881
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100882 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ?
883 MODE32_hyp : MODE32_svc;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100884
885 /*
886 * TODO: Choose async. exception bits if HYP mode is not
887 * implemented according to the values of SCR.{AW, FW} bits
888 */
Vikram Kanigiri9851e422014-05-13 14:42:08 +0100889 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT;
890
Jimmy Brissoned202072020-08-04 16:18:52 -0500891 ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee,
892 daif);
Achin Gupta4f6ad662013-10-25 09:08:21 +0100893 }
894
Andrew Thoelke4e126072014-06-04 21:10:52 +0100895 return PSCI_E_SUCCESS;
Achin Gupta4f6ad662013-10-25 09:08:21 +0100896}
Julius Werner8e0ef0f2019-07-09 14:02:43 -0700897#else /* !__aarch64__ */
898static int psci_get_ns_ep_info(entry_point_info_t *ep,
899 uintptr_t entrypoint,
900 u_register_t context_id)
901{
902 u_register_t ep_attr;
903 unsigned int aif, ee, mode;
904 u_register_t scr = read_scr();
905 u_register_t ns_sctlr, sctlr;
906
907 /* Switch to non secure state */
908 write_scr(scr | SCR_NS_BIT);
909 isb();
910 ns_sctlr = read_sctlr();
911
912 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr;
913
914 /* Return to original state */
915 write_scr(scr);
916 isb();
917 ee = 0;
918
919 ep_attr = NON_SECURE | EP_ST_DISABLE;
920 if (sctlr & SCTLR_EE_BIT) {
921 ep_attr |= EP_EE_BIG;
922 ee = 1;
923 }
924 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr);
925
926 ep->pc = entrypoint;
927 zeromem(&ep->args, sizeof(ep->args));
928 ep->args.arg0 = context_id;
929
930 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc;
931
932 /*
933 * TODO: Choose async. exception bits if HYP mode is not
934 * implemented according to the values of SCR.{AW, FW} bits
935 */
936 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT;
937
938 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif);
939
940 return PSCI_E_SUCCESS;
941}
942
943#endif /* __aarch64__ */
Achin Gupta4f6ad662013-10-25 09:08:21 +0100944
945/*******************************************************************************
Soby Mathewf1f97a12015-07-15 12:13:26 +0100946 * This function validates the entrypoint with the platform layer if the
947 * appropriate pm_ops hook is exported by the platform and returns the
948 * 'entry_point_info'.
949 ******************************************************************************/
950int psci_validate_entry_point(entry_point_info_t *ep,
Soby Mathew011ca182015-07-29 17:05:03 +0100951 uintptr_t entrypoint,
952 u_register_t context_id)
Soby Mathewf1f97a12015-07-15 12:13:26 +0100953{
954 int rc;
955
956 /* Validate the entrypoint using platform psci_ops */
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100957 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) {
Soby Mathewf1f97a12015-07-15 12:13:26 +0100958 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint);
959 if (rc != PSCI_E_SUCCESS)
960 return PSCI_E_INVALID_ADDRESS;
961 }
962
963 /*
964 * Verify and derive the re-entry information for
965 * the non-secure world from the non-secure state from
966 * where this call originated.
967 */
968 rc = psci_get_ns_ep_info(ep, entrypoint, context_id);
969 return rc;
970}
971
972/*******************************************************************************
Achin Gupta4f6ad662013-10-25 09:08:21 +0100973 * Generic handler which is called when a cpu is physically powered on. It
Soby Mathew981487a2015-07-13 14:10:57 +0100974 * traverses the node information and finds the highest power level powered
975 * off and performs generic, architectural, platform setup and state management
976 * to power on that power level and power levels below it.
977 * e.g. For a cpu that's been powered on, it will call the platform specific
978 * code to enable the gic cpu interface and for a cluster it will enable
979 * coherency at the interconnect level in addition to gic cpu interface.
Achin Gupta4f6ad662013-10-25 09:08:21 +0100980 ******************************************************************************/
Soby Mathewd0194872016-04-29 19:01:30 +0100981void psci_warmboot_entrypoint(void)
Achin Gupta4f6ad662013-10-25 09:08:21 +0100982{
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +0100983 unsigned int end_pwrlvl;
Deepika Bhavnani79ffab52019-08-27 00:32:24 +0300984 unsigned int cpu_idx = plat_my_core_pos();
Andrew F. Davis74e89782019-06-04 10:46:54 -0400985 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
Soby Mathew981487a2015-07-13 14:10:57 +0100986 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} };
Achin Gupta4f6ad662013-10-25 09:08:21 +0100987
Boyan Karatotev36cebf92023-03-08 11:56:49 +0000988 /* Init registers that never change for the lifetime of TF-A */
989 cm_manage_extensions_el3();
990
Achin Gupta4f6ad662013-10-25 09:08:21 +0100991 /*
Soby Mathew981487a2015-07-13 14:10:57 +0100992 * Verify that we have been explicitly turned ON or resumed from
993 * suspend.
Achin Gupta4f6ad662013-10-25 09:08:21 +0100994 */
Soby Mathew981487a2015-07-13 14:10:57 +0100995 if (psci_get_aff_info_state() == AFF_STATE_OFF) {
Andrew Walbran8fe72b92020-01-23 16:22:44 +0000996 ERROR("Unexpected affinity info state.\n");
James Morrissey40a6f642014-02-10 14:24:36 +0000997 panic();
Soby Mathew981487a2015-07-13 14:10:57 +0100998 }
Achin Gupta4f6ad662013-10-25 09:08:21 +0100999
1000 /*
Soby Mathew981487a2015-07-13 14:10:57 +01001001 * Get the maximum power domain level to traverse to after this cpu
1002 * has been physically powered up.
Achin Gupta4f6ad662013-10-25 09:08:21 +01001003 */
Soby Mathew981487a2015-07-13 14:10:57 +01001004 end_pwrlvl = get_power_on_target_pwrlvl();
Achin Guptaf6b9e992014-07-31 11:19:11 +01001005
Andrew F. Davis74e89782019-06-04 10:46:54 -04001006 /* Get the parent nodes */
1007 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes);
1008
Achin Guptaf6b9e992014-07-31 11:19:11 +01001009 /*
Soby Mathew981487a2015-07-13 14:10:57 +01001010 * This function acquires the lock corresponding to each power level so
1011 * that by the time all locks are taken, the system topology is snapshot
1012 * and state management can be done safely.
Achin Guptaf6b9e992014-07-31 11:19:11 +01001013 */
Andrew F. Davis74e89782019-06-04 10:46:54 -04001014 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes);
Achin Guptaf6b9e992014-07-31 11:19:11 +01001015
Soby Mathew8336f682017-10-16 15:19:31 +01001016 psci_get_target_local_pwr_states(end_pwrlvl, &state_info);
1017
Yatharth Kochar241ec6c2016-05-09 18:26:35 +01001018#if ENABLE_PSCI_STAT
dp-arm66abfbe2017-01-31 13:01:04 +00001019 plat_psci_stat_accounting_stop(&state_info);
Yatharth Kochar241ec6c2016-05-09 18:26:35 +01001020#endif
1021
Achin Gupta4f6ad662013-10-25 09:08:21 +01001022 /*
Soby Mathew981487a2015-07-13 14:10:57 +01001023 * This CPU could be resuming from suspend or it could have just been
1024 * turned on. To distinguish between these 2 cases, we examine the
1025 * affinity state of the CPU:
1026 * - If the affinity state is ON_PENDING then it has just been
1027 * turned on.
1028 * - Else it is resuming from suspend.
1029 *
1030 * Depending on the type of warm reset identified, choose the right set
1031 * of power management handler and perform the generic, architecture
1032 * and platform specific handling.
Achin Guptacab78e42014-07-28 00:09:01 +01001033 */
Soby Mathew981487a2015-07-13 14:10:57 +01001034 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING)
1035 psci_cpu_on_finish(cpu_idx, &state_info);
1036 else
1037 psci_cpu_suspend_finish(cpu_idx, &state_info);
Achin Guptacab78e42014-07-28 00:09:01 +01001038
1039 /*
Soby Mathew981487a2015-07-13 14:10:57 +01001040 * Set the requested and target state of this CPU and all the higher
1041 * power domains which are ancestors of this CPU to run.
Achin Guptaf6b9e992014-07-31 11:19:11 +01001042 */
Soby Mathew981487a2015-07-13 14:10:57 +01001043 psci_set_pwr_domains_to_run(end_pwrlvl);
Achin Guptaf6b9e992014-07-31 11:19:11 +01001044
Yatharth Kochar241ec6c2016-05-09 18:26:35 +01001045#if ENABLE_PSCI_STAT
1046 /*
1047 * Update PSCI stats.
1048 * Caches are off when writing stats data on the power down path.
1049 * Since caches are now enabled, it's necessary to do cache
1050 * maintenance before reading that same data.
1051 */
dp-arm66abfbe2017-01-31 13:01:04 +00001052 psci_stats_update_pwr_up(end_pwrlvl, &state_info);
Yatharth Kochar241ec6c2016-05-09 18:26:35 +01001053#endif
1054
Achin Guptaf6b9e992014-07-31 11:19:11 +01001055 /*
Soby Mathew981487a2015-07-13 14:10:57 +01001056 * This loop releases the lock corresponding to each power level
Achin Gupta0959db52013-12-02 17:33:04 +00001057 * in the reverse order to which they were acquired.
1058 */
Andrew F. Davis74e89782019-06-04 10:46:54 -04001059 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes);
Achin Gupta4f6ad662013-10-25 09:08:21 +01001060}
Jeenu Viswambharan7f366602014-02-20 17:11:00 +00001061
1062/*******************************************************************************
1063 * This function initializes the set of hooks that PSCI invokes as part of power
1064 * management operation. The power management hooks are expected to be provided
1065 * by the SPD, after it finishes all its initialization
1066 ******************************************************************************/
Dan Handleye2712bc2014-04-10 15:37:22 +01001067void psci_register_spd_pm_hook(const spd_pm_ops_t *pm)
Jeenu Viswambharan7f366602014-02-20 17:11:00 +00001068{
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001069 assert(pm != NULL);
Jeenu Viswambharan7f366602014-02-20 17:11:00 +00001070 psci_spd_pm = pm;
Soby Mathew6cdddaf2015-01-07 11:10:22 +00001071
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001072 if (pm->svc_migrate != NULL)
Soby Mathew6cdddaf2015-01-07 11:10:22 +00001073 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64);
1074
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001075 if (pm->svc_migrate_info != NULL)
Soby Mathew6cdddaf2015-01-07 11:10:22 +00001076 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64)
1077 | define_psci_cap(PSCI_MIG_INFO_TYPE);
Jeenu Viswambharan7f366602014-02-20 17:11:00 +00001078}
Juan Castillo4dc4a472014-08-12 11:17:06 +01001079
1080/*******************************************************************************
Soby Mathew110fe362014-10-23 10:35:34 +01001081 * This function invokes the migrate info hook in the spd_pm_ops. It performs
1082 * the necessary return value validation. If the Secure Payload is UP and
1083 * migrate capable, it returns the mpidr of the CPU on which the Secure payload
1084 * is resident through the mpidr parameter. Else the value of the parameter on
1085 * return is undefined.
1086 ******************************************************************************/
Soby Mathew011ca182015-07-29 17:05:03 +01001087int psci_spd_migrate_info(u_register_t *mpidr)
Soby Mathew110fe362014-10-23 10:35:34 +01001088{
1089 int rc;
1090
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001091 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL))
Soby Mathew110fe362014-10-23 10:35:34 +01001092 return PSCI_E_NOT_SUPPORTED;
1093
1094 rc = psci_spd_pm->svc_migrate_info(mpidr);
1095
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001096 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) ||
1097 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED));
Soby Mathew110fe362014-10-23 10:35:34 +01001098
1099 return rc;
1100}
1101
1102
1103/*******************************************************************************
Soby Mathew981487a2015-07-13 14:10:57 +01001104 * This function prints the state of all power domains present in the
Juan Castillo4dc4a472014-08-12 11:17:06 +01001105 * system
1106 ******************************************************************************/
Soby Mathew981487a2015-07-13 14:10:57 +01001107void psci_print_power_domain_map(void)
Juan Castillo4dc4a472014-08-12 11:17:06 +01001108{
1109#if LOG_LEVEL >= LOG_LEVEL_INFO
Pankaj Gupta02c35682019-10-15 15:44:45 +05301110 unsigned int idx;
Soby Mathew981487a2015-07-13 14:10:57 +01001111 plat_local_state_t state;
1112 plat_local_state_type_t state_type;
1113
Juan Castillo4dc4a472014-08-12 11:17:06 +01001114 /* This array maps to the PSCI_STATE_X definitions in psci.h */
Soby Mathew24ab34f2016-05-03 17:11:42 +01001115 static const char * const psci_state_type_str[] = {
Juan Castillo4dc4a472014-08-12 11:17:06 +01001116 "ON",
Soby Mathew981487a2015-07-13 14:10:57 +01001117 "RETENTION",
Juan Castillo4dc4a472014-08-12 11:17:06 +01001118 "OFF",
Juan Castillo4dc4a472014-08-12 11:17:06 +01001119 };
1120
Soby Mathew981487a2015-07-13 14:10:57 +01001121 INFO("PSCI Power Domain Map:\n");
Pankaj Gupta02c35682019-10-15 15:44:45 +05301122 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count);
Soby Mathew981487a2015-07-13 14:10:57 +01001123 idx++) {
1124 state_type = find_local_state_type(
1125 psci_non_cpu_pd_nodes[idx].local_state);
Yann Gautier507e0cd2022-02-14 11:09:23 +01001126 INFO(" Domain Node : Level %u, parent_node %u,"
Soby Mathew981487a2015-07-13 14:10:57 +01001127 " State %s (0x%x)\n",
1128 psci_non_cpu_pd_nodes[idx].level,
1129 psci_non_cpu_pd_nodes[idx].parent_node,
1130 psci_state_type_str[state_type],
1131 psci_non_cpu_pd_nodes[idx].local_state);
1132 }
1133
Pankaj Gupta02c35682019-10-15 15:44:45 +05301134 for (idx = 0; idx < psci_plat_core_count; idx++) {
Soby Mathew981487a2015-07-13 14:10:57 +01001135 state = psci_get_cpu_local_state_by_idx(idx);
1136 state_type = find_local_state_type(state);
Yann Gautier507e0cd2022-02-14 11:09:23 +01001137 INFO(" CPU Node : MPID 0x%llx, parent_node %u,"
Soby Mathew981487a2015-07-13 14:10:57 +01001138 " State %s (0x%x)\n",
Soby Mathewa0fedc42016-06-16 14:52:04 +01001139 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr,
Soby Mathew981487a2015-07-13 14:10:57 +01001140 psci_cpu_pd_nodes[idx].parent_node,
1141 psci_state_type_str[state_type],
1142 psci_get_cpu_local_state_by_idx(idx));
Juan Castillo4dc4a472014-08-12 11:17:06 +01001143 }
1144#endif
1145}
Soby Mathew981487a2015-07-13 14:10:57 +01001146
Jeenu Viswambharanbc1a9292017-02-16 14:55:15 +00001147/******************************************************************************
1148 * Return whether any secondaries were powered up with CPU_ON call. A CPU that
1149 * have ever been powered up would have set its MPDIR value to something other
1150 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to
1151 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is
1152 * meaningful only when called on the primary CPU during early boot.
1153 *****************************************************************************/
1154int psci_secondaries_brought_up(void)
1155{
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001156 unsigned int idx, n_valid = 0U;
Jeenu Viswambharanbc1a9292017-02-16 14:55:15 +00001157
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001158 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) {
Jeenu Viswambharanbc1a9292017-02-16 14:55:15 +00001159 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR)
1160 n_valid++;
1161 }
1162
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001163 assert(n_valid > 0U);
Jeenu Viswambharanbc1a9292017-02-16 14:55:15 +00001164
Antonio Nino Diaz78a95a62018-07-17 15:10:08 +01001165 return (n_valid > 1U) ? 1 : 0;
Jeenu Viswambharanbc1a9292017-02-16 14:55:15 +00001166}
1167
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +00001168/*******************************************************************************
1169 * Initiate power down sequence, by calling power down operations registered for
1170 * this CPU.
1171 ******************************************************************************/
Pranav Madhuc1e61d02022-07-22 23:11:16 +05301172void psci_pwrdown_cpu(unsigned int power_level)
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +00001173{
1174#if HW_ASSISTED_COHERENCY
1175 /*
1176 * With hardware-assisted coherency, the CPU drivers only initiate the
1177 * power down sequence, without performing cache-maintenance operations
Andrew F. Davis564f9542018-08-30 12:08:01 -05001178 * in software. Data caches enabled both before and after this call.
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +00001179 */
1180 prepare_cpu_pwr_dwn(power_level);
1181#else
1182 /*
1183 * Without hardware-assisted coherency, the CPU drivers disable data
Andrew F. Davis564f9542018-08-30 12:08:01 -05001184 * caches, then perform cache-maintenance operations in software.
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +00001185 *
Andrew F. Davis564f9542018-08-30 12:08:01 -05001186 * This also calls prepare_cpu_pwr_dwn() to initiate power down
1187 * sequence, but that function will return with data caches disabled.
1188 * We must ensure that the stack memory is flushed out to memory before
1189 * we start popping from it again.
Jeenu Viswambharan346bfd82017-01-05 11:01:02 +00001190 */
1191 psci_do_pwrdown_cache_maintenance(power_level);
1192#endif
1193}
Sandeep Tripathy12030042020-08-17 20:22:13 +05301194
1195/*******************************************************************************
1196 * This function invokes the callback 'stop_func()' with the 'mpidr' of each
1197 * online PE. Caller can pass suitable method to stop a remote core.
1198 *
1199 * 'wait_ms' is the timeout value in milliseconds for the other cores to
1200 * transition to power down state. Passing '0' makes it non-blocking.
1201 *
1202 * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the
1203 * given timeout.
1204 ******************************************************************************/
1205int psci_stop_other_cores(unsigned int wait_ms,
1206 void (*stop_func)(u_register_t mpidr))
1207{
1208 unsigned int idx, this_cpu_idx;
1209
1210 this_cpu_idx = plat_my_core_pos();
1211
1212 /* Invoke stop_func for each core */
1213 for (idx = 0U; idx < psci_plat_core_count; idx++) {
1214 /* skip current CPU */
1215 if (idx == this_cpu_idx) {
1216 continue;
1217 }
1218
1219 /* Check if the CPU is ON */
1220 if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) {
1221 (*stop_func)(psci_cpu_pd_nodes[idx].mpidr);
1222 }
1223 }
1224
1225 /* Need to wait for other cores to shutdown */
1226 if (wait_ms != 0U) {
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001227 while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu())) {
Sandeep Tripathy12030042020-08-17 20:22:13 +05301228 mdelay(1U);
1229 }
1230
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001231 if (!psci_is_last_on_cpu()) {
Sandeep Tripathy12030042020-08-17 20:22:13 +05301232 WARN("Failed to stop all cores!\n");
1233 psci_print_power_domain_map();
1234 return PSCI_E_DENIED;
1235 }
1236 }
1237
1238 return PSCI_E_SUCCESS;
1239}
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001240
1241/*******************************************************************************
1242 * This function verifies that all the other cores in the system have been
1243 * turned OFF and the current CPU is the last running CPU in the system.
1244 * Returns true if the current CPU is the last ON CPU or false otherwise.
1245 *
1246 * This API has following differences with psci_is_last_on_cpu
1247 * 1. PSCI states are locked
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001248 ******************************************************************************/
1249bool psci_is_last_on_cpu_safe(void)
1250{
1251 unsigned int this_core = plat_my_core_pos();
1252 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001253
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001254 psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes);
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001255
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001256 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001257
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001258 if (!psci_is_last_on_cpu()) {
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001259 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001260 return false;
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001261 }
1262
Jayanth Dodderi Chidanand70763502022-08-22 23:46:10 +01001263 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
1264
Lucian Paul-Trifu5e685352022-03-02 21:28:24 +00001265 return true;
1266}
Wing Li71f69df2022-09-14 13:18:15 -07001267
1268/*******************************************************************************
1269 * This function verifies that all cores in the system have been turned ON.
1270 * Returns true, if all CPUs are ON or false otherwise.
1271 *
1272 * This API has following differences with psci_are_all_cpus_on
1273 * 1. PSCI states are locked
1274 ******************************************************************************/
1275bool psci_are_all_cpus_on_safe(void)
1276{
1277 unsigned int this_core = plat_my_core_pos();
1278 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
1279
1280 psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes);
1281
1282 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
1283
1284 if (!psci_are_all_cpus_on()) {
1285 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
1286 return false;
1287 }
1288
1289 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
1290
1291 return true;
1292}