Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 1 | /* |
Antonio Nino Diaz | 5a42b68 | 2018-07-18 11:57:21 +0100 | [diff] [blame] | 2 | * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 3 | * |
dp-arm | fa3cf0b | 2017-05-03 09:38:09 +0100 | [diff] [blame] | 4 | * SPDX-License-Identifier: BSD-3-Clause |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 5 | */ |
| 6 | |
| 7 | #include <arch.h> |
| 8 | #include <arch_helpers.h> |
| 9 | #include <assert.h> |
| 10 | #include <bl_common.h> |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 11 | #include <context_mgmt.h> |
Isla Mitchell | 9930501 | 2017-07-11 14:54:08 +0100 | [diff] [blame] | 12 | #include <debug.h> |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 13 | #include <platform.h> |
Jeenu Viswambharan | 55e56a9 | 2017-09-22 08:32:10 +0100 | [diff] [blame] | 14 | #include <pubsub_events.h> |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 15 | #include <stddef.h> |
| 16 | #include "psci_private.h" |
| 17 | |
Antonio Nino Diaz | 5a42b68 | 2018-07-18 11:57:21 +0100 | [diff] [blame] | 18 | /* |
| 19 | * Helper functions for the CPU level spinlocks |
| 20 | */ |
| 21 | static inline void psci_spin_lock_cpu(int idx) |
| 22 | { |
| 23 | spin_lock(&psci_cpu_pd_nodes[idx].cpu_lock); |
| 24 | } |
| 25 | |
| 26 | static inline void psci_spin_unlock_cpu(int idx) |
| 27 | { |
| 28 | spin_unlock(&psci_cpu_pd_nodes[idx].cpu_lock); |
| 29 | } |
| 30 | |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 31 | /******************************************************************************* |
| 32 | * This function checks whether a cpu which has been requested to be turned on |
| 33 | * is OFF to begin with. |
| 34 | ******************************************************************************/ |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 35 | static int cpu_on_validate_state(aff_info_state_t aff_state) |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 36 | { |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 37 | if (aff_state == AFF_STATE_ON) |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 38 | return PSCI_E_ALREADY_ON; |
| 39 | |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 40 | if (aff_state == AFF_STATE_ON_PENDING) |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 41 | return PSCI_E_ON_PENDING; |
| 42 | |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 43 | assert(aff_state == AFF_STATE_OFF); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 44 | return PSCI_E_SUCCESS; |
| 45 | } |
| 46 | |
| 47 | /******************************************************************************* |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 48 | * Generic handler which is called to physically power on a cpu identified by |
Soby Mathew | 6b8b302 | 2015-06-30 11:00:24 +0100 | [diff] [blame] | 49 | * its mpidr. It performs the generic, architectural, platform setup and state |
| 50 | * management to power on the target cpu e.g. it will ensure that |
| 51 | * enough information is stashed for it to resume execution in the non-secure |
| 52 | * security state. |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 53 | * |
Soby Mathew | 3a9e8bf | 2015-05-05 16:33:16 +0100 | [diff] [blame] | 54 | * The state of all the relevant power domains are changed after calling the |
Soby Mathew | 6b8b302 | 2015-06-30 11:00:24 +0100 | [diff] [blame] | 55 | * platform handler as it can return error. |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 56 | ******************************************************************************/ |
Soby Mathew | 011ca18 | 2015-07-29 17:05:03 +0100 | [diff] [blame] | 57 | int psci_cpu_on_start(u_register_t target_cpu, |
Sandrine Bailleux | 7497bff | 2016-04-25 09:28:43 +0100 | [diff] [blame] | 58 | entry_point_info_t *ep) |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 59 | { |
| 60 | int rc; |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 61 | unsigned int target_idx = plat_core_pos_by_mpidr(target_cpu); |
Soby Mathew | ca37050 | 2016-01-26 11:47:53 +0000 | [diff] [blame] | 62 | aff_info_state_t target_aff_state; |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 63 | |
Sandrine Bailleux | 6181acb | 2016-04-22 13:00:19 +0100 | [diff] [blame] | 64 | /* Calling function must supply valid input arguments */ |
| 65 | assert((int) target_idx >= 0); |
| 66 | assert(ep != NULL); |
| 67 | |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 68 | /* |
| 69 | * This function must only be called on platforms where the |
| 70 | * CPU_ON platform hooks have been implemented. |
| 71 | */ |
Soby Mathew | 3a9e8bf | 2015-05-05 16:33:16 +0100 | [diff] [blame] | 72 | assert(psci_plat_pm_ops->pwr_domain_on && |
| 73 | psci_plat_pm_ops->pwr_domain_on_finish); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 74 | |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 75 | /* Protect against multiple CPUs trying to turn ON the same target CPU */ |
| 76 | psci_spin_lock_cpu(target_idx); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 77 | |
| 78 | /* |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 79 | * Generic management: Ensure that the cpu is off to be |
| 80 | * turned on. |
David Cunado | 06adba2 | 2017-07-19 12:14:07 +0100 | [diff] [blame] | 81 | * Perform cache maintanence ahead of reading the target CPU state to |
| 82 | * ensure that the data is not stale. |
| 83 | * There is a theoretical edge case where the cache may contain stale |
| 84 | * data for the target CPU data - this can occur under the following |
| 85 | * conditions: |
| 86 | * - the target CPU is in another cluster from the current |
| 87 | * - the target CPU was the last CPU to shutdown on its cluster |
| 88 | * - the cluster was removed from coherency as part of the CPU shutdown |
| 89 | * |
| 90 | * In this case the cache maintenace that was performed as part of the |
| 91 | * target CPUs shutdown was not seen by the current CPU's cluster. And |
| 92 | * so the cache may contain stale data for the target CPU. |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 93 | */ |
David Cunado | 06adba2 | 2017-07-19 12:14:07 +0100 | [diff] [blame] | 94 | flush_cpu_data_by_index(target_idx, psci_svc_cpu_data.aff_info_state); |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 95 | rc = cpu_on_validate_state(psci_get_aff_info_state_by_idx(target_idx)); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 96 | if (rc != PSCI_E_SUCCESS) |
| 97 | goto exit; |
| 98 | |
| 99 | /* |
| 100 | * Call the cpu on handler registered by the Secure Payload Dispatcher |
| 101 | * to let it do any bookeeping. If the handler encounters an error, it's |
| 102 | * expected to assert within |
| 103 | */ |
| 104 | if (psci_spd_pm && psci_spd_pm->svc_on) |
| 105 | psci_spd_pm->svc_on(target_cpu); |
| 106 | |
| 107 | /* |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 108 | * Set the Affinity info state of the target cpu to ON_PENDING. |
Soby Mathew | ca37050 | 2016-01-26 11:47:53 +0000 | [diff] [blame] | 109 | * Flush aff_info_state as it will be accessed with caches |
| 110 | * turned OFF. |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 111 | */ |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 112 | psci_set_aff_info_state_by_idx(target_idx, AFF_STATE_ON_PENDING); |
Soby Mathew | ca37050 | 2016-01-26 11:47:53 +0000 | [diff] [blame] | 113 | flush_cpu_data_by_index(target_idx, psci_svc_cpu_data.aff_info_state); |
| 114 | |
| 115 | /* |
| 116 | * The cache line invalidation by the target CPU after setting the |
| 117 | * state to OFF (see psci_do_cpu_off()), could cause the update to |
| 118 | * aff_info_state to be invalidated. Retry the update if the target |
| 119 | * CPU aff_info_state is not ON_PENDING. |
| 120 | */ |
| 121 | target_aff_state = psci_get_aff_info_state_by_idx(target_idx); |
| 122 | if (target_aff_state != AFF_STATE_ON_PENDING) { |
| 123 | assert(target_aff_state == AFF_STATE_OFF); |
| 124 | psci_set_aff_info_state_by_idx(target_idx, AFF_STATE_ON_PENDING); |
| 125 | flush_cpu_data_by_index(target_idx, psci_svc_cpu_data.aff_info_state); |
| 126 | |
| 127 | assert(psci_get_aff_info_state_by_idx(target_idx) == AFF_STATE_ON_PENDING); |
| 128 | } |
Soby Mathew | 6b8b302 | 2015-06-30 11:00:24 +0100 | [diff] [blame] | 129 | |
| 130 | /* |
| 131 | * Perform generic, architecture and platform specific handling. |
| 132 | */ |
Soby Mathew | 6b8b302 | 2015-06-30 11:00:24 +0100 | [diff] [blame] | 133 | /* |
| 134 | * Plat. management: Give the platform the current state |
| 135 | * of the target cpu to allow it to perform the necessary |
| 136 | * steps to power on. |
| 137 | */ |
Soby Mathew | 011ca18 | 2015-07-29 17:05:03 +0100 | [diff] [blame] | 138 | rc = psci_plat_pm_ops->pwr_domain_on(target_cpu); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 139 | assert(rc == PSCI_E_SUCCESS || rc == PSCI_E_INTERN_FAIL); |
| 140 | |
| 141 | if (rc == PSCI_E_SUCCESS) |
| 142 | /* Store the re-entry information for the non-secure world. */ |
Soby Mathew | b0082d2 | 2015-04-09 13:40:55 +0100 | [diff] [blame] | 143 | cm_init_context_by_index(target_idx, ep); |
Soby Mathew | ca37050 | 2016-01-26 11:47:53 +0000 | [diff] [blame] | 144 | else { |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 145 | /* Restore the state on error. */ |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 146 | psci_set_aff_info_state_by_idx(target_idx, AFF_STATE_OFF); |
Soby Mathew | ca37050 | 2016-01-26 11:47:53 +0000 | [diff] [blame] | 147 | flush_cpu_data_by_index(target_idx, psci_svc_cpu_data.aff_info_state); |
| 148 | } |
Soby Mathew | b0082d2 | 2015-04-09 13:40:55 +0100 | [diff] [blame] | 149 | |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 150 | exit: |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 151 | psci_spin_unlock_cpu(target_idx); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 152 | return rc; |
| 153 | } |
| 154 | |
| 155 | /******************************************************************************* |
Soby Mathew | 3a9e8bf | 2015-05-05 16:33:16 +0100 | [diff] [blame] | 156 | * The following function finish an earlier power on request. They |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 157 | * are called by the common finisher routine in psci_common.c. The `state_info` |
| 158 | * is the psci_power_state from which this CPU has woken up from. |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 159 | ******************************************************************************/ |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 160 | void psci_cpu_on_finish(unsigned int cpu_idx, |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 161 | psci_power_state_t *state_info) |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 162 | { |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 163 | /* |
| 164 | * Plat. management: Perform the platform specific actions |
| 165 | * for this cpu e.g. enabling the gic or zeroing the mailbox |
| 166 | * register. The actual state of this cpu has already been |
| 167 | * changed. |
| 168 | */ |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 169 | psci_plat_pm_ops->pwr_domain_on_finish(state_info); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 170 | |
Soby Mathew | 043fe9c | 2017-04-10 22:35:42 +0100 | [diff] [blame] | 171 | #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 172 | /* |
| 173 | * Arch. management: Enable data cache and manage stack memory |
| 174 | */ |
| 175 | psci_do_pwrup_cache_maintenance(); |
Jeenu Viswambharan | 346bfd8 | 2017-01-05 11:01:02 +0000 | [diff] [blame] | 176 | #endif |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 177 | |
| 178 | /* |
| 179 | * All the platform specific actions for turning this cpu |
| 180 | * on have completed. Perform enough arch.initialization |
| 181 | * to run in the non-secure address space. |
| 182 | */ |
Soby Mathew | d019487 | 2016-04-29 19:01:30 +0100 | [diff] [blame] | 183 | psci_arch_setup(); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 184 | |
| 185 | /* |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 186 | * Lock the CPU spin lock to make sure that the context initialization |
| 187 | * is done. Since the lock is only used in this function to create |
| 188 | * a synchronization point with cpu_on_start(), it can be released |
| 189 | * immediately. |
| 190 | */ |
| 191 | psci_spin_lock_cpu(cpu_idx); |
| 192 | psci_spin_unlock_cpu(cpu_idx); |
| 193 | |
Soby Mathew | 85dbf5a | 2015-04-07 12:16:56 +0100 | [diff] [blame] | 194 | /* Ensure we have been explicitly woken up by another cpu */ |
| 195 | assert(psci_get_aff_info_state() == AFF_STATE_ON_PENDING); |
| 196 | |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 197 | /* |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 198 | * Call the cpu on finish handler registered by the Secure Payload |
| 199 | * Dispatcher to let it do any bookeeping. If the handler encounters an |
| 200 | * error, it's expected to assert within |
| 201 | */ |
| 202 | if (psci_spd_pm && psci_spd_pm->svc_on_finish) |
| 203 | psci_spd_pm->svc_on_finish(0); |
| 204 | |
Jeenu Viswambharan | 55e56a9 | 2017-09-22 08:32:10 +0100 | [diff] [blame] | 205 | PUBLISH_EVENT(psci_cpu_on_finish); |
| 206 | |
Soby Mathew | 9d754f6 | 2015-04-08 17:42:06 +0100 | [diff] [blame] | 207 | /* Populate the mpidr field within the cpu node array */ |
| 208 | /* This needs to be done only once */ |
| 209 | psci_cpu_pd_nodes[cpu_idx].mpidr = read_mpidr() & MPIDR_AFFINITY_MASK; |
| 210 | |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 211 | /* |
| 212 | * Generic management: Now we just need to retrieve the |
| 213 | * information that we had stashed away during the cpu_on |
| 214 | * call to set this cpu on its way. |
| 215 | */ |
| 216 | cm_prepare_el3_exit(NON_SECURE); |
Soby Mathew | 991d42c | 2015-06-29 16:30:12 +0100 | [diff] [blame] | 217 | } |