perf(psci): pass my_core_pos around instead of calling it repeatedly
On some platforms plat_my_core_pos is a nontrivial function that takes a
bit of time and the compiler really doesn't like to inline. In the PSCI
library, at least, we have no need to keep repeatedly calling it and we
can instead pass it around as an argument. This saves on a lot of
redundant calls, speeding the library up a bit.
Change-Id: I137f69bea80d7cac90d7a20ffe98e1ba8d77246f
Signed-off-by: Boyan Karatotev <boyan.karatotev@arm.com>
diff --git a/lib/psci/psci_common.c b/lib/psci/psci_common.c
index 5a9a015..93d71b8 100644
--- a/lib/psci/psci_common.c
+++ b/lib/psci/psci_common.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -172,9 +172,9 @@
* Returns 1 (true) if the current CPU is the last ON CPU or 0 (false)
* otherwise.
******************************************************************************/
-static bool psci_is_last_cpu_to_idle_at_pwrlvl(unsigned int end_pwrlvl)
+static bool psci_is_last_cpu_to_idle_at_pwrlvl(unsigned int my_idx, unsigned int end_pwrlvl)
{
- unsigned int my_idx, lvl;
+ unsigned int lvl;
unsigned int parent_idx = 0;
unsigned int cpu_start_idx, ncpus, cpu_idx;
plat_local_state_t local_state;
@@ -183,7 +183,6 @@
return true;
}
- my_idx = plat_my_core_pos();
parent_idx = psci_cpu_pd_nodes[my_idx].parent_node;
for (lvl = PSCI_CPU_PWR_LVL + U(1); lvl < end_pwrlvl; lvl++) {
parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
@@ -214,11 +213,9 @@
* turned OFF and the current CPU is the last running CPU in the system.
* Returns true, if the current CPU is the last ON CPU or false otherwise.
******************************************************************************/
-bool psci_is_last_on_cpu(void)
+bool psci_is_last_on_cpu(unsigned int my_idx)
{
- unsigned int cpu_idx, my_idx = plat_my_core_pos();
-
- for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) {
+ for (unsigned int cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) {
if (cpu_idx == my_idx) {
assert(psci_get_aff_info_state() == AFF_STATE_ON);
continue;
@@ -426,14 +423,14 @@
* function will be called after a cpu is powered on to find the local state
* each power domain has emerged from.
*****************************************************************************/
-void psci_get_target_local_pwr_states(unsigned int end_pwrlvl,
+void psci_get_target_local_pwr_states(unsigned int cpu_idx, unsigned int end_pwrlvl,
psci_power_state_t *target_state)
{
unsigned int parent_idx, lvl;
plat_local_state_t *pd_state = target_state->pwr_domain_state;
pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state();
- parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
+ parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
/* Copy the local power state from node to state_info */
for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
@@ -452,7 +449,7 @@
* enter. This function will be called after coordination of requested power
* states has been done for each power level.
*****************************************************************************/
-void psci_set_target_local_pwr_states(unsigned int end_pwrlvl,
+void psci_set_target_local_pwr_states(unsigned int cpu_idx, unsigned int end_pwrlvl,
const psci_power_state_t *target_state)
{
unsigned int parent_idx, lvl;
@@ -466,7 +463,7 @@
*/
psci_flush_cpu_data(psci_svc_cpu_data.local_state);
- parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
+ parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
/* Copy the local_state from state_info */
for (lvl = 1U; lvl <= end_pwrlvl; lvl++) {
@@ -498,9 +495,9 @@
* affinity info state, target power state and requested power state for the
* current CPU and all its ancestor power domains to RUN.
*****************************************************************************/
-void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl)
+void psci_set_pwr_domains_to_run(unsigned int cpu_idx, unsigned int end_pwrlvl)
{
- unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl;
+ unsigned int parent_idx, lvl;
parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
/* Reset the local_state to RUN for the non cpu power domains. */
@@ -542,10 +539,10 @@
* This function will only be invoked with data cache enabled and while
* powering down a core.
*****************************************************************************/
-void psci_do_state_coordination(unsigned int end_pwrlvl,
+void psci_do_state_coordination(unsigned int cpu_idx, unsigned int end_pwrlvl,
psci_power_state_t *state_info)
{
- unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos();
+ unsigned int lvl, parent_idx;
unsigned int start_idx;
unsigned int ncpus;
plat_local_state_t target_state, *req_states;
@@ -618,11 +615,11 @@
* This function will only be invoked with data cache enabled and while
* powering down a core.
*****************************************************************************/
-int psci_validate_state_coordination(unsigned int end_pwrlvl,
+int psci_validate_state_coordination(unsigned int cpu_idx, unsigned int end_pwrlvl,
psci_power_state_t *state_info)
{
int rc = PSCI_E_SUCCESS;
- unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos();
+ unsigned int lvl, parent_idx;
unsigned int start_idx;
unsigned int ncpus;
plat_local_state_t target_state, *req_states;
@@ -672,7 +669,7 @@
* specified power level.
*/
lvl = state_info->last_at_pwrlvl;
- if (!psci_is_last_cpu_to_idle_at_pwrlvl(lvl)) {
+ if (!psci_is_last_cpu_to_idle_at_pwrlvl(cpu_idx, lvl)) {
rc = PSCI_E_DENIED;
}
@@ -1002,7 +999,7 @@
*/
psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes);
- psci_get_target_local_pwr_states(end_pwrlvl, &state_info);
+ psci_get_target_local_pwr_states(cpu_idx, end_pwrlvl, &state_info);
#if ENABLE_PSCI_STAT
plat_psci_stat_accounting_stop(&state_info);
@@ -1036,10 +1033,10 @@
* Set the requested and target state of this CPU and all the higher
* power domains which are ancestors of this CPU to run.
*/
- psci_set_pwr_domains_to_run(end_pwrlvl);
+ psci_set_pwr_domains_to_run(cpu_idx, end_pwrlvl);
#if ENABLE_PSCI_STAT
- psci_stats_update_pwr_up(end_pwrlvl, &state_info);
+ psci_stats_update_pwr_up(cpu_idx, end_pwrlvl, &state_info);
#endif
/*
@@ -1209,15 +1206,11 @@
* The function returns 'PSCI_E_DENIED' if some cores failed to stop within the
* given timeout.
******************************************************************************/
-int psci_stop_other_cores(unsigned int wait_ms,
+int psci_stop_other_cores(unsigned int this_cpu_idx, unsigned int wait_ms,
void (*stop_func)(u_register_t mpidr))
{
- unsigned int idx, this_cpu_idx;
-
- this_cpu_idx = plat_my_core_pos();
-
/* Invoke stop_func for each core */
- for (idx = 0U; idx < psci_plat_core_count; idx++) {
+ for (unsigned int idx = 0U; idx < psci_plat_core_count; idx++) {
/* skip current CPU */
if (idx == this_cpu_idx) {
continue;
@@ -1231,11 +1224,11 @@
/* Need to wait for other cores to shutdown */
if (wait_ms != 0U) {
- while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu())) {
+ while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu(this_cpu_idx))) {
mdelay(1U);
}
- if (!psci_is_last_on_cpu()) {
+ if (!psci_is_last_on_cpu(this_cpu_idx)) {
WARN("Failed to stop all cores!\n");
psci_print_power_domain_map();
return PSCI_E_DENIED;
@@ -1253,16 +1246,15 @@
* This API has following differences with psci_is_last_on_cpu
* 1. PSCI states are locked
******************************************************************************/
-bool psci_is_last_on_cpu_safe(void)
+bool psci_is_last_on_cpu_safe(unsigned int this_core)
{
- unsigned int this_core = plat_my_core_pos();
unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes);
psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
- if (!psci_is_last_on_cpu()) {
+ if (!psci_is_last_on_cpu(this_core)) {
psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
return false;
}
@@ -1279,9 +1271,8 @@
* This API has following differences with psci_are_all_cpus_on
* 1. PSCI states are locked
******************************************************************************/
-bool psci_are_all_cpus_on_safe(void)
+bool psci_are_all_cpus_on_safe(unsigned int this_core)
{
- unsigned int this_core = plat_my_core_pos();
unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes);
diff --git a/lib/psci/psci_main.c b/lib/psci/psci_main.c
index a015531..7ac0e02 100644
--- a/lib/psci/psci_main.c
+++ b/lib/psci/psci_main.c
@@ -59,8 +59,8 @@
entry_point_info_t ep;
psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} };
plat_local_state_t cpu_pd_state;
-#if PSCI_OS_INIT_MODE
unsigned int cpu_idx = plat_my_core_pos();
+#if PSCI_OS_INIT_MODE
plat_local_state_t prev[PLAT_MAX_PWR_LVL];
#endif
@@ -145,7 +145,7 @@
plat_psci_stat_accounting_stop(&state_info);
/* Update PSCI stats */
- psci_stats_update_pwr_up(PSCI_CPU_PWR_LVL, &state_info);
+ psci_stats_update_pwr_up(cpu_idx, PSCI_CPU_PWR_LVL, &state_info);
#endif
return PSCI_E_SUCCESS;
@@ -167,7 +167,8 @@
* might return if the power down was abandoned for any reason, e.g.
* arrival of an interrupt
*/
- rc = psci_cpu_suspend_start(&ep,
+ rc = psci_cpu_suspend_start(cpu_idx,
+ &ep,
target_pwrlvl,
&state_info,
is_power_down_state);
@@ -181,9 +182,10 @@
int rc;
psci_power_state_t state_info;
entry_point_info_t ep;
+ unsigned int cpu_idx = plat_my_core_pos();
/* Check if the current CPU is the last ON CPU in the system */
- if (!psci_is_last_on_cpu())
+ if (!psci_is_last_on_cpu(cpu_idx))
return PSCI_E_DENIED;
/* Validate the entry point and get the entry_point_info */
@@ -212,7 +214,8 @@
* might return if the power down was abandoned for any reason, e.g.
* arrival of an interrupt
*/
- rc = psci_cpu_suspend_start(&ep,
+ rc = psci_cpu_suspend_start(cpu_idx,
+ &ep,
PLAT_MAX_PWR_LVL,
&state_info,
PSTATE_TYPE_POWERDOWN);
@@ -399,9 +402,11 @@
return PSCI_E_SUCCESS;
}
+ unsigned int this_core = plat_my_core_pos();
+
if (mode == PLAT_COORD) {
/* Check if the current CPU is the last ON CPU in the system */
- if (!psci_is_last_on_cpu_safe()) {
+ if (!psci_is_last_on_cpu_safe(this_core)) {
return PSCI_E_DENIED;
}
}
@@ -411,8 +416,8 @@
* Check if all CPUs in the system are ON or if the current
* CPU is the last ON CPU in the system.
*/
- if (!(psci_are_all_cpus_on_safe() ||
- psci_is_last_on_cpu_safe())) {
+ if (!(psci_are_all_cpus_on_safe(this_core) ||
+ psci_is_last_on_cpu_safe(this_core))) {
return PSCI_E_DENIED;
}
}
diff --git a/lib/psci/psci_off.c b/lib/psci/psci_off.c
index a13ae03..d40ee3f 100644
--- a/lib/psci/psci_off.c
+++ b/lib/psci/psci_off.c
@@ -102,14 +102,14 @@
* it returns the negotiated state info for each power level upto
* the end level specified.
*/
- psci_do_state_coordination(end_pwrlvl, &state_info);
+ psci_do_state_coordination(idx, end_pwrlvl, &state_info);
/* Update the target state in the power domain nodes */
- psci_set_target_local_pwr_states(end_pwrlvl, &state_info);
+ psci_set_target_local_pwr_states(idx, end_pwrlvl, &state_info);
#if ENABLE_PSCI_STAT
/* Update the last cpu for each level till end_pwrlvl */
- psci_stats_update_pwr_down(end_pwrlvl, &state_info);
+ psci_stats_update_pwr_down(idx, end_pwrlvl, &state_info);
#endif
/*
diff --git a/lib/psci/psci_private.h b/lib/psci/psci_private.h
index e63275f..6622755 100644
--- a/lib/psci/psci_private.h
+++ b/lib/psci/psci_private.h
@@ -295,19 +295,19 @@
void psci_restore_req_local_pwr_states(unsigned int cpu_idx,
plat_local_state_t *prev);
#endif
-void psci_get_target_local_pwr_states(unsigned int end_pwrlvl,
+void psci_get_target_local_pwr_states(unsigned int cpu_idx, unsigned int end_pwrlvl,
psci_power_state_t *target_state);
-void psci_set_target_local_pwr_states(unsigned int end_pwrlvl,
+void psci_set_target_local_pwr_states(unsigned int cpu_idx, unsigned int end_pwrlvl,
const psci_power_state_t *target_state);
int psci_validate_entry_point(entry_point_info_t *ep,
uintptr_t entrypoint, u_register_t context_id);
void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx,
unsigned int end_lvl,
unsigned int *node_index);
-void psci_do_state_coordination(unsigned int end_pwrlvl,
+void psci_do_state_coordination(unsigned int cpu_idx, unsigned int end_pwrlvl,
psci_power_state_t *state_info);
#if PSCI_OS_INIT_MODE
-int psci_validate_state_coordination(unsigned int end_pwrlvl,
+int psci_validate_state_coordination(unsigned int cpu_idx, unsigned int end_pwrlvl,
psci_power_state_t *state_info);
#endif
void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl,
@@ -318,9 +318,9 @@
unsigned int is_power_down_state);
unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info);
unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info);
-void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl);
+void psci_set_pwr_domains_to_run(unsigned int cpu_idx, unsigned int end_pwrlvl);
void psci_print_power_domain_map(void);
-bool psci_is_last_on_cpu(void);
+bool psci_is_last_on_cpu(unsigned int my_idx);
int psci_spd_migrate_info(u_register_t *mpidr);
/*
@@ -343,7 +343,8 @@
int psci_do_cpu_off(unsigned int end_pwrlvl);
/* Private exported functions from psci_suspend.c */
-int psci_cpu_suspend_start(const entry_point_info_t *ep,
+int psci_cpu_suspend_start(unsigned int idx,
+ const entry_point_info_t *ep,
unsigned int end_pwrlvl,
psci_power_state_t *state_info,
unsigned int is_power_down_state);
@@ -360,9 +361,9 @@
u_register_t psci_system_reset2(uint32_t reset_type, u_register_t cookie);
/* Private exported functions from psci_stat.c */
-void psci_stats_update_pwr_down(unsigned int end_pwrlvl,
+void psci_stats_update_pwr_down(unsigned int cpu_idx, unsigned int end_pwrlvl,
const psci_power_state_t *state_info);
-void psci_stats_update_pwr_up(unsigned int end_pwrlvl,
+void psci_stats_update_pwr_up(unsigned int cpu_idx, unsigned int end_pwrlvl,
const psci_power_state_t *state_info);
u_register_t psci_stat_residency(u_register_t target_cpu,
unsigned int power_state);
diff --git a/lib/psci/psci_setup.c b/lib/psci/psci_setup.c
index 1afc918..a81ba4a 100644
--- a/lib/psci/psci_setup.c
+++ b/lib/psci/psci_setup.c
@@ -202,6 +202,7 @@
int __init psci_setup(const psci_lib_args_t *lib_args)
{
const unsigned char *topology_tree;
+ unsigned int cpu_idx = plat_my_core_pos();
assert(VERIFY_PSCI_LIB_ARGS_V1(lib_args));
@@ -218,7 +219,7 @@
psci_update_pwrlvl_limits();
/* Populate the mpidr field of cpu node for this CPU */
- psci_cpu_pd_nodes[plat_my_core_pos()].mpidr =
+ psci_cpu_pd_nodes[cpu_idx].mpidr =
read_mpidr() & MPIDR_AFFINITY_MASK;
psci_init_req_local_pwr_states();
@@ -227,7 +228,7 @@
* Set the requested and target state of this CPU and all the higher
* power domain levels for this CPU to run.
*/
- psci_set_pwr_domains_to_run(PLAT_MAX_PWR_LVL);
+ psci_set_pwr_domains_to_run(cpu_idx, PLAT_MAX_PWR_LVL);
(void) plat_setup_psci_ops((uintptr_t)lib_args->mailbox_ep,
&psci_plat_pm_ops);
diff --git a/lib/psci/psci_stat.c b/lib/psci/psci_stat.c
index bedb816..1225557 100644
--- a/lib/psci/psci_stat.c
+++ b/lib/psci/psci_stat.c
@@ -73,11 +73,10 @@
* This function will only be invoked with data cache enabled and while
* powering down a core.
******************************************************************************/
-void psci_stats_update_pwr_down(unsigned int end_pwrlvl,
+void psci_stats_update_pwr_down(unsigned int cpu_idx, unsigned int end_pwrlvl,
const psci_power_state_t *state_info)
{
unsigned int lvl, parent_idx;
- unsigned int cpu_idx = plat_my_core_pos();
assert(end_pwrlvl <= PLAT_MAX_PWR_LVL);
assert(state_info != NULL);
@@ -106,11 +105,10 @@
* and NON-CPU power domains.
* It is called with caches enabled and locks acquired(for NON-CPU domain)
******************************************************************************/
-void psci_stats_update_pwr_up(unsigned int end_pwrlvl,
+void psci_stats_update_pwr_up(unsigned int cpu_idx, unsigned int end_pwrlvl,
const psci_power_state_t *state_info)
{
unsigned int lvl, parent_idx;
- unsigned int cpu_idx = plat_my_core_pos();
int stat_idx;
plat_local_state_t local_state;
u_register_t residency;
diff --git a/lib/psci/psci_suspend.c b/lib/psci/psci_suspend.c
index c3d6954..2aadbfd 100644
--- a/lib/psci/psci_suspend.c
+++ b/lib/psci/psci_suspend.c
@@ -118,14 +118,14 @@
* the state transition has been done, no further error is expected and it is
* not possible to undo any of the actions taken beyond that point.
******************************************************************************/
-int psci_cpu_suspend_start(const entry_point_info_t *ep,
+int psci_cpu_suspend_start(unsigned int idx,
+ const entry_point_info_t *ep,
unsigned int end_pwrlvl,
psci_power_state_t *state_info,
unsigned int is_power_down_state)
{
int rc = PSCI_E_SUCCESS;
bool skip_wfi = false;
- unsigned int idx = plat_my_core_pos();
unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
/*
@@ -161,7 +161,7 @@
* This function validates the requested state info for
* OS-initiated mode.
*/
- rc = psci_validate_state_coordination(end_pwrlvl, state_info);
+ rc = psci_validate_state_coordination(idx, end_pwrlvl, state_info);
if (rc != PSCI_E_SUCCESS) {
skip_wfi = true;
goto exit;
@@ -173,7 +173,7 @@
* it returns the negotiated state info for each power level upto
* the end level specified.
*/
- psci_do_state_coordination(end_pwrlvl, state_info);
+ psci_do_state_coordination(idx, end_pwrlvl, state_info);
#if PSCI_OS_INIT_MODE
}
#endif
@@ -189,11 +189,11 @@
#endif
/* Update the target state in the power domain nodes */
- psci_set_target_local_pwr_states(end_pwrlvl, state_info);
+ psci_set_target_local_pwr_states(idx, end_pwrlvl, state_info);
#if ENABLE_PSCI_STAT
/* Update the last cpu for each level till end_pwrlvl */
- psci_stats_update_pwr_down(end_pwrlvl, state_info);
+ psci_stats_update_pwr_down(idx, end_pwrlvl, state_info);
#endif
if (is_power_down_state != 0U)
@@ -269,11 +269,11 @@
* 'end_pwrlvl'. The exit retention state could be deeper than the entry
* state as a result of state coordination amongst other CPUs post wfi.
*/
- psci_get_target_local_pwr_states(end_pwrlvl, state_info);
+ psci_get_target_local_pwr_states(idx, end_pwrlvl, state_info);
#if ENABLE_PSCI_STAT
plat_psci_stat_accounting_stop(state_info);
- psci_stats_update_pwr_up(end_pwrlvl, state_info);
+ psci_stats_update_pwr_up(idx, end_pwrlvl, state_info);
#endif
/*
@@ -286,7 +286,7 @@
* Set the requested and target state of this CPU and all the higher
* power domain levels for this CPU to run.
*/
- psci_set_pwr_domains_to_run(end_pwrlvl);
+ psci_set_pwr_domains_to_run(idx, end_pwrlvl);
psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes);