common/psci/psci_common.c - filogic/atf - Gitiles

 /*
  * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  * Redistributions of source code must retain the above copyright notice, this
  * list of conditions and the following disclaimer.
  *
  * Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
  *
  * Neither the name of ARM nor the names of its contributors may be used
  * to endorse or promote products derived from this software without specific
  * prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */

 #include <stdio.h>
 #include <string.h>
 #include <assert.h>
 #include <arch_helpers.h>
 #include <console.h>
 #include <platform.h>
 #include <psci.h>
 #include <psci_private.h>
 #include <runtime_svc.h>

 /*******************************************************************************
  * Arrays that contains information needs to resume a cpu's execution when woken
  * out of suspend or off states. 'psci_ns_einfo_idx' keeps track of the next
  * free index in the 'psci_ns_entry_info' & 'psci_secure_context' arrays. Each
  * cpu is allocated a single entry in each array during startup.
  ******************************************************************************/
 secure_context psci_secure_context[PSCI_NUM_AFFS];
 ns_entry_info psci_ns_entry_info[PSCI_NUM_AFFS];
 unsigned int psci_ns_einfo_idx;

 /*******************************************************************************
  * Grand array that holds the platform's topology information for state
  * management of affinity instances. Each node (aff_map_node) in the array
  * corresponds to an affinity instance e.g. cluster, cpu within an mpidr
  ******************************************************************************/
 aff_map_node psci_aff_map[PSCI_NUM_AFFS]
 __attribute__ ((section("tzfw_coherent_mem")));

 /*******************************************************************************
  * In a system, a certain number of affinity instances are present at an
  * affinity level. The cumulative number of instances across all levels are
  * stored in 'psci_aff_map'. The topology tree has been flattenned into this
  * array. To retrieve nodes, information about the extents of each affinity
  * level i.e. start index and end index needs to be present. 'psci_aff_limits'
  * stores this information.
  ******************************************************************************/
 aff_limits_node psci_aff_limits[MPIDR_MAX_AFFLVL + 1];

 /*******************************************************************************
  * Pointer to functions exported by the platform to complete power mgmt. ops
  ******************************************************************************/
 plat_pm_ops *psci_plat_pm_ops;

 /*******************************************************************************
  * Simple routine to retrieve the maximum affinity level supported by the
  * platform and check that it makes sense.
  ******************************************************************************/
 int get_max_afflvl()
 {
 	int aff_lvl;

 	aff_lvl = plat_get_max_afflvl();
 	assert(aff_lvl <= MPIDR_MAX_AFFLVL && aff_lvl >= MPIDR_AFFLVL0);

 	return aff_lvl;
 }

 /*******************************************************************************
  * Simple routine to set the id of an affinity instance at a given level in the
  * mpidr.
  ******************************************************************************/
 unsigned long mpidr_set_aff_inst(unsigned long mpidr,
 				 unsigned char aff_inst,
 				 int aff_lvl)
 {
 	unsigned long aff_shift;

 	assert(aff_lvl <= MPIDR_AFFLVL3);

 	/*
 	 * Decide the number of bits to shift by depending upon
 	 * the affinity level
 	 */
 	aff_shift = get_afflvl_shift(aff_lvl);

 	/* Clear the existing affinity instance & set the new one*/
 	mpidr &= ~(MPIDR_AFFLVL_MASK << aff_shift);
 	mpidr |= aff_inst << aff_shift;

 	return mpidr;
 }

 /*******************************************************************************
  * This function sanity checks a range of affinity levels.
  ******************************************************************************/
 int psci_check_afflvl_range(int start_afflvl, int end_afflvl)
 {
 	/* Sanity check the parameters passed */
 	if (end_afflvl > MPIDR_MAX_AFFLVL)
 		return PSCI_E_INVALID_PARAMS;

 	if (start_afflvl < MPIDR_AFFLVL0)
 		return PSCI_E_INVALID_PARAMS;

 	if (end_afflvl < start_afflvl)
 		return PSCI_E_INVALID_PARAMS;

 	return PSCI_E_SUCCESS;
 }

 /*******************************************************************************
  * This function is passed an array of pointers to affinity level nodes in the
  * topology tree for an mpidr. It picks up locks for each affinity level bottom
  * up in the range specified.
  ******************************************************************************/
 void psci_acquire_afflvl_locks(unsigned long mpidr,
 			       int start_afflvl,
 			       int end_afflvl,
 			       mpidr_aff_map_nodes mpidr_nodes)
 {
 	int level;

 	for (level = start_afflvl; level <= end_afflvl; level++) {
 		if (mpidr_nodes[level] == NULL)
 			continue;
 		bakery_lock_get(mpidr, &mpidr_nodes[level]->lock);
 	}
 }

 /*******************************************************************************
  * This function is passed an array of pointers to affinity level nodes in the
  * topology tree for an mpidr. It releases the lock for each affinity level top
  * down in the range specified.
  ******************************************************************************/
 void psci_release_afflvl_locks(unsigned long mpidr,
 			       int start_afflvl,
 			       int end_afflvl,
 			       mpidr_aff_map_nodes mpidr_nodes)
 {
 	int level;

 	for (level = end_afflvl; level >= start_afflvl; level--) {
 		if (mpidr_nodes[level] == NULL)
 			continue;
 		bakery_lock_release(mpidr, &mpidr_nodes[level]->lock);
 	}
 }

 /*******************************************************************************
  * Simple routine to determine whether an affinity instance at a given level
  * in an mpidr exists or not.
  ******************************************************************************/
 int psci_validate_mpidr(unsigned long mpidr, int level)
 {
 	aff_map_node *node;

 	node = psci_get_aff_map_node(mpidr, level);
 	if (node && (node->state & PSCI_AFF_PRESENT))
 		return PSCI_E_SUCCESS;
 	else
 		return PSCI_E_INVALID_PARAMS;
 }

 /*******************************************************************************
  * Simple routine to determine the first affinity level instance that is present
  * between the start and end affinity levels. This helps to skip handling of
  * absent affinity levels while performing psci operations.
  * The start level can be > or <= to the end level depending upon whether this
  * routine is expected to search top down or bottom up.
  ******************************************************************************/
 int psci_get_first_present_afflvl(unsigned long mpidr,
 				  int start_afflvl,
 				  int end_afflvl,
 				  aff_map_node **node)
 {
 	int level;

 	/* Check whether we have to search up or down */
 	if (start_afflvl <= end_afflvl) {
 		for (level = start_afflvl; level <= end_afflvl; level++) {
 			*node = psci_get_aff_map_node(mpidr, level);
 			if (*node && ((*node)->state & PSCI_AFF_PRESENT))
 				break;
 		}
 	} else {
 		for (level = start_afflvl; level >= end_afflvl; level--) {
 			*node = psci_get_aff_map_node(mpidr, level);
 			if (*node && ((*node)->state & PSCI_AFF_PRESENT))
 				break;
 		}
 	}

 	return level;
 }

 /*******************************************************************************
  * Iteratively change the affinity state between the current and target affinity
  * levels. The target state matters only if we are starting from affinity level
  * 0 i.e. a cpu otherwise the state depends upon the state of the lower affinity
  * levels.
  ******************************************************************************/
 int psci_change_state(mpidr_aff_map_nodes mpidr_nodes,
 		      int start_afflvl,
 		      int end_afflvl,
 		      unsigned int tgt_state)
 {
 	int rc = PSCI_E_SUCCESS, level;
 	unsigned int state;
 	aff_map_node *node;

 	/*
 	 * Get a temp pointer to the node. It is not possible that affinity
 	 * level 0 is missing. Simply ignore higher missing levels.
 	 */
 	for (level = start_afflvl; level <= end_afflvl; level++) {

 		node = mpidr_nodes[level];
 		if (level == MPIDR_AFFLVL0) {
 			assert(node);
 			psci_set_state(node->state, tgt_state);
 		} else {
 			if (node == NULL)
 				continue;
 			state = psci_calculate_affinity_state(node);
 			psci_set_state(node->state, state);
 		}
 	}

 	/* If all went well then the cpu should be in the target state */
 	if (start_afflvl == MPIDR_AFFLVL0) {
 		node = mpidr_nodes[MPIDR_AFFLVL0];
 		state = psci_get_state(node->state);
 		assert(tgt_state == state);
 	}

 	return rc;
 }

 /*******************************************************************************
  * This routine does the heavy lifting for psci_change_state(). It examines the
  * state of each affinity instance at the next lower affinity level and decides
  * its final state accordingly. If a lower affinity instance is ON then the
  * higher affinity instance is ON. If all the lower affinity instances are OFF
  * then the higher affinity instance is OFF. If atleast one lower affinity
  * instance is SUSPENDED then the higher affinity instance is SUSPENDED. If only
  * a single lower affinity instance is ON_PENDING then the higher affinity
  * instance in ON_PENDING as well.
  ******************************************************************************/
 unsigned int psci_calculate_affinity_state(aff_map_node *aff_node)
 {
 	int ctr;
 	unsigned int aff_count, hi_aff_state;
 	unsigned long tempidr;
 	aff_map_node *lo_aff_node;

 	/* Cannot calculate lowest affinity state. It is simply assigned */
 	assert(aff_node->level > MPIDR_AFFLVL0);

 	/*
 	 * Find the number of affinity instances at level X-1 e.g. number of
 	 * cpus in a cluster. The level X state depends upon the state of each
 	 * instance at level X-1
 	 */
 	hi_aff_state = PSCI_STATE_OFF;
 	aff_count = plat_get_aff_count(aff_node->level - 1, aff_node->mpidr);
 	for (ctr = 0; ctr < aff_count; ctr++) {

 		/*
 		 * Create a mpidr for each lower affinity level (X-1). Use their
 		 * states to influence the higher affinity state (X).
 		 */
 		tempidr = mpidr_set_aff_inst(aff_node->mpidr,
 					     ctr,
 					     aff_node->level - 1);
 		lo_aff_node = psci_get_aff_map_node(tempidr,
 						    aff_node->level - 1);
 		assert(lo_aff_node);

 		/* Continue only if the cpu exists within the cluster */
 		if (!(lo_aff_node->state & PSCI_AFF_PRESENT))
 			continue;

 		switch (psci_get_state(lo_aff_node->state)) {

 		/*
 		 * If any lower affinity is on within the cluster, then
 		 * the higher affinity is on.
 		 */
 		case PSCI_STATE_ON:
 			return PSCI_STATE_ON;

 		/*
 		 * At least one X-1 needs to be suspended for X to be suspended
 		 * but it is effectively on for the affinity_info call.
 		 * SUSPEND > ON_PENDING > OFF.
 		 */
 		case PSCI_STATE_SUSPEND:
 			hi_aff_state = PSCI_STATE_SUSPEND;
 			continue;

 		/*
 		 * Atleast one X-1 needs to be on_pending & the rest off for X
 		 * to be on_pending. ON_PENDING > OFF.
 		 */
 		case PSCI_STATE_ON_PENDING:
 			if (hi_aff_state != PSCI_STATE_SUSPEND)
 				hi_aff_state = PSCI_STATE_ON_PENDING;
 			continue;

 		/* Higher affinity is off if all lower affinities are off. */
 		case PSCI_STATE_OFF:
 			continue;

 		default:
 			assert(0);
 		}
 	}

 	return hi_aff_state;
 }

 /*******************************************************************************
  * This function retrieves all the stashed information needed to correctly
  * resume a cpu's execution in the non-secure state after it has been physically
  * powered on i.e. turned ON or resumed from SUSPEND
  ******************************************************************************/
 void psci_get_ns_entry_info(unsigned int index)
 {
 	unsigned long sctlr = 0, scr, el_status, id_aa64pfr0;
 	gp_regs *ns_gp_regs;

 	scr = read_scr();

 	/* Switch to the non-secure view of the registers */
 	write_scr(scr | SCR_NS_BIT);

 	/* Find out which EL we are going to */
 	id_aa64pfr0 = read_id_aa64pfr0_el1();
 	el_status = (id_aa64pfr0 >> ID_AA64PFR0_EL2_SHIFT) &
 		ID_AA64PFR0_ELX_MASK;

 	/* Restore endianess */
 	if (psci_ns_entry_info[index].sctlr & SCTLR_EE_BIT)
 		sctlr |= SCTLR_EE_BIT;
 	else
 		sctlr &= ~SCTLR_EE_BIT;

 	/* Turn off MMU and Caching */
 	sctlr &= ~(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_M_BIT);

 	/* Set the register width */
 	if (psci_ns_entry_info[index].scr & SCR_RW_BIT)
 		scr |= SCR_RW_BIT;
 	else
 		scr &= ~SCR_RW_BIT;

 	scr |= SCR_NS_BIT;

 	if (el_status)
 		write_sctlr_el2(sctlr);
 	else
 		write_sctlr_el1(sctlr);

 	/* Fulfill the cpu_on entry reqs. as per the psci spec */
 	write_scr(scr);
 	write_elr(psci_ns_entry_info[index].eret_info.entrypoint);

 	/*
 	 * Set the general purpose registers to ~0 upon entry into the
 	 * non-secure world except for x0 which should contain the
 	 * context id & spsr. This is done directly on the "would be"
 	 * stack pointer. Prior to entry into the non-secure world, an
 	 * offset equivalent to the size of the 'gp_regs' structure is
 	 * added to the sp. This general purpose register context is
 	 * retrieved then.
 	 */
 	ns_gp_regs = (gp_regs *) platform_get_stack(read_mpidr());
 	ns_gp_regs--;
 	memset(ns_gp_regs, ~0, sizeof(*ns_gp_regs));
 	ns_gp_regs->x0 = psci_ns_entry_info[index].context_id;
 	ns_gp_regs->spsr = psci_ns_entry_info[index].eret_info.spsr;
 }

 /*******************************************************************************
  * This function retrieves and stashes all the information needed to correctly
  * resume a cpu's execution in the non-secure state after it has been physically
  * powered on i.e. turned ON or resumed from SUSPEND. This is done prior to
  * turning it on or before suspending it.
  ******************************************************************************/
 int psci_set_ns_entry_info(unsigned int index,
 			   unsigned long entrypoint,
 			   unsigned long context_id)
 {
 	int rc = PSCI_E_SUCCESS;
 	unsigned int rw, mode, ee, spsr = 0;
 	unsigned long id_aa64pfr0 = read_id_aa64pfr0_el1(), scr = read_scr();
 	unsigned long el_status;

 	/* Figure out what mode do we enter the non-secure world in */
 	el_status = (id_aa64pfr0 >> ID_AA64PFR0_EL2_SHIFT) &
 		ID_AA64PFR0_ELX_MASK;

 	/*
 	 * Figure out whether the cpu enters the non-secure address space
 	 * in aarch32 or aarch64
 	 */
 	rw = scr & SCR_RW_BIT;
 	if (rw) {

 		/*
 		 * Check whether a Thumb entry point has been provided for an
 		 * aarch64 EL
 		 */
 		if (entrypoint & 0x1)
 			return PSCI_E_INVALID_PARAMS;

 		if (el_status && (scr & SCR_HCE_BIT)) {
 			mode = MODE_EL2;
 			ee = read_sctlr_el2() & SCTLR_EE_BIT;
 		} else {
 			mode = MODE_EL1;
 			ee = read_sctlr_el1() & SCTLR_EE_BIT;
 		}

 		spsr = DAIF_DBG_BIT | DAIF_ABT_BIT;
 		spsr |= DAIF_IRQ_BIT | DAIF_FIQ_BIT;
 		spsr <<= PSR_DAIF_SHIFT;
 		spsr |= make_spsr(mode, MODE_SP_ELX, !rw);

 		psci_ns_entry_info[index].sctlr |= ee;
 		psci_ns_entry_info[index].scr |= SCR_RW_BIT;
 	} else {

 		/* Check whether aarch32 has to be entered in Thumb mode */
 		if (entrypoint & 0x1)
 			spsr = SPSR32_T_BIT;

 		if (el_status && (scr & SCR_HCE_BIT)) {
 			mode = AARCH32_MODE_HYP;
 			ee = read_sctlr_el2() & SCTLR_EE_BIT;
 		} else {
 			mode = AARCH32_MODE_SVC;
 			ee = read_sctlr_el1() & SCTLR_EE_BIT;
 		}

 		/*
 		 * TODO: Choose async. exception bits if HYP mode is not
 		 * implemented according to the values of SCR.{AW, FW} bits
 		 */
 		spsr |= DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT;
 		spsr <<= PSR_DAIF_SHIFT;
 		if(ee)
 			spsr |= SPSR32_EE_BIT;
 		spsr |= mode;

 		/* Ensure that the CSPR.E and SCTLR.EE bits match */
 		psci_ns_entry_info[index].sctlr |= ee;
 		psci_ns_entry_info[index].scr &= ~SCR_RW_BIT;
 	}

 	psci_ns_entry_info[index].eret_info.entrypoint = entrypoint;
 	psci_ns_entry_info[index].eret_info.spsr = spsr;
 	psci_ns_entry_info[index].context_id = context_id;

 	return rc;
 }

 /*******************************************************************************
  * An affinity level could be on, on_pending, suspended or off. These are the
  * logical states it can be in. Physically either it is off or on. When it is in
  * the state on_pending then it is about to be turned on. It is not possible to
  * tell whether that's actually happenned or not. So we err on the side of
  * caution & treat the affinity level as being turned off.
  ******************************************************************************/
 inline unsigned int psci_get_phys_state(unsigned int aff_state)
 {
 	return (aff_state != PSCI_STATE_ON ? PSCI_STATE_OFF : PSCI_STATE_ON);
 }

 unsigned int psci_get_aff_phys_state(aff_map_node *aff_node)
 {
 	unsigned int aff_state;

 	aff_state = psci_get_state(aff_node->state);
 	return psci_get_phys_state(aff_state);
 }

 /*******************************************************************************
  * This function takes an array of pointers to affinity instance nodes in the
  * topology tree and calls the physical power on handler for the corresponding
  * affinity levels
  ******************************************************************************/
 static int psci_call_power_on_handlers(mpidr_aff_map_nodes mpidr_nodes,
 				       int start_afflvl,
 				       int end_afflvl,
 				       afflvl_power_on_finisher *pon_handlers,
 				       unsigned long mpidr)
 {
 	int rc = PSCI_E_INVALID_PARAMS, level;
 	aff_map_node *node;

 	for (level = end_afflvl; level >= start_afflvl; level--) {
 		node = mpidr_nodes[level];
 		if (node == NULL)
 			continue;

 		/*
 		 * If we run into any trouble while powering up an
 		 * affinity instance, then there is no recovery path
 		 * so simply return an error and let the caller take
 		 * care of the situation.
 		 */
 		rc = pon_handlers[level](mpidr, node);
 		if (rc != PSCI_E_SUCCESS)
 			break;
 	}

 	return rc;
 }

 /*******************************************************************************
  * Generic handler which is called when a cpu is physically powered on. It
  * traverses through all the affinity levels performing generic, architectural,
  * platform setup and state management e.g. for a cluster that's been powered
  * on, it will call the platform specific code which will enable coherency at
  * the interconnect level. For a cpu it could mean turning on the MMU etc.
  *
  * The state of all the relevant affinity levels is changed after calling the
  * affinity level specific handlers as their actions would depend upon the state
  * the affinity level is exiting from.
  *
  * The affinity level specific handlers are called in descending order i.e. from
  * the highest to the lowest affinity level implemented by the platform because
  * to turn on affinity level X it is neccesary to turn on affinity level X + 1
  * first.
  *
  * CAUTION: This function is called with coherent stacks so that coherency and
  * the mmu can be turned on safely.
  ******************************************************************************/
 void psci_afflvl_power_on_finish(unsigned long mpidr,
 				 int start_afflvl,
 				 int end_afflvl,
 				 afflvl_power_on_finisher *pon_handlers)
 {
 	mpidr_aff_map_nodes mpidr_nodes;
 	int rc;

 	mpidr &= MPIDR_AFFINITY_MASK;;

 	/*
 	 * Collect the pointers to the nodes in the topology tree for
 	 * each affinity instance in the mpidr. If this function does
 	 * not return successfully then either the mpidr or the affinity
 	 * levels are incorrect. Either case is an irrecoverable error.
 	 */
 	rc = psci_get_aff_map_nodes(mpidr,
 				    start_afflvl,
 				    end_afflvl,
 				    mpidr_nodes);
 	assert (rc == PSCI_E_SUCCESS);

 	/*
 	 * This function acquires the lock corresponding to each affinity
 	 * level so that by the time all locks are taken, the system topology
 	 * is snapshot and state management can be done safely.
 	 */
 	psci_acquire_afflvl_locks(mpidr,
 				  start_afflvl,
 				  end_afflvl,
 				  mpidr_nodes);

 	/* Perform generic, architecture and platform specific handling */
 	rc = psci_call_power_on_handlers(mpidr_nodes,
 					 start_afflvl,
 					 end_afflvl,
 					 pon_handlers,
 					 mpidr);
 	assert (rc == PSCI_E_SUCCESS);

 	/*
 	 * State management: Update the state of each affinity instance
 	 * between the start and end affinity levels
 	 */
 	psci_change_state(mpidr_nodes,
 			  start_afflvl,
 			  end_afflvl,
 			  PSCI_STATE_ON);

 	/*
 	 * This loop releases the lock corresponding to each affinity level
 	 * in the reverse order to which they were acquired.
 	 */
 	psci_release_afflvl_locks(mpidr,
 				  start_afflvl,
 				  end_afflvl,
 				  mpidr_nodes);
 }
	/*
	* Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are met:
	*
	* Redistributions of source code must retain the above copyright notice, this
	* list of conditions and the following disclaimer.
	*
	* Redistributions in binary form must reproduce the above copyright notice,
	* this list of conditions and the following disclaimer in the documentation
	* and/or other materials provided with the distribution.
	*
	* Neither the name of ARM nor the names of its contributors may be used
	* to endorse or promote products derived from this software without specific
	* prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <stdio.h>
	#include <string.h>
	#include <assert.h>
	#include <arch_helpers.h>
	#include <console.h>
	#include <platform.h>
	#include <psci.h>
	#include <psci_private.h>
	#include <runtime_svc.h>

	/*******************************************************************************
	* Arrays that contains information needs to resume a cpu's execution when woken
	* out of suspend or off states. 'psci_ns_einfo_idx' keeps track of the next
	* free index in the 'psci_ns_entry_info' & 'psci_secure_context' arrays. Each
	* cpu is allocated a single entry in each array during startup.
	******************************************************************************/
	secure_context psci_secure_context[PSCI_NUM_AFFS];
	ns_entry_info psci_ns_entry_info[PSCI_NUM_AFFS];
	unsigned int psci_ns_einfo_idx;

	/*******************************************************************************
	* Grand array that holds the platform's topology information for state
	* management of affinity instances. Each node (aff_map_node) in the array
	* corresponds to an affinity instance e.g. cluster, cpu within an mpidr
	******************************************************************************/
	aff_map_node psci_aff_map[PSCI_NUM_AFFS]
	__attribute__ ((section("tzfw_coherent_mem")));

	/*******************************************************************************
	* In a system, a certain number of affinity instances are present at an
	* affinity level. The cumulative number of instances across all levels are
	* stored in 'psci_aff_map'. The topology tree has been flattenned into this
	* array. To retrieve nodes, information about the extents of each affinity
	* level i.e. start index and end index needs to be present. 'psci_aff_limits'
	* stores this information.
	******************************************************************************/
	aff_limits_node psci_aff_limits[MPIDR_MAX_AFFLVL + 1];

	/*******************************************************************************
	* Pointer to functions exported by the platform to complete power mgmt. ops
	******************************************************************************/
	plat_pm_ops *psci_plat_pm_ops;

	/*******************************************************************************
	* Simple routine to retrieve the maximum affinity level supported by the
	* platform and check that it makes sense.
	******************************************************************************/
	int get_max_afflvl()
	{
	int aff_lvl;

	aff_lvl = plat_get_max_afflvl();
	assert(aff_lvl <= MPIDR_MAX_AFFLVL && aff_lvl >= MPIDR_AFFLVL0);

	return aff_lvl;
	}

	/*******************************************************************************
	* Simple routine to set the id of an affinity instance at a given level in the
	* mpidr.
	******************************************************************************/
	unsigned long mpidr_set_aff_inst(unsigned long mpidr,
	unsigned char aff_inst,
	int aff_lvl)
	{
	unsigned long aff_shift;

	assert(aff_lvl <= MPIDR_AFFLVL3);

	/*
	* Decide the number of bits to shift by depending upon
	* the affinity level
	*/
	aff_shift = get_afflvl_shift(aff_lvl);

	/* Clear the existing affinity instance & set the new one*/
	mpidr &= ~(MPIDR_AFFLVL_MASK << aff_shift);
	mpidr \|= aff_inst << aff_shift;

	return mpidr;
	}

	/*******************************************************************************
	* This function sanity checks a range of affinity levels.
	******************************************************************************/
	int psci_check_afflvl_range(int start_afflvl, int end_afflvl)
	{
	/* Sanity check the parameters passed */
	if (end_afflvl > MPIDR_MAX_AFFLVL)
	return PSCI_E_INVALID_PARAMS;

	if (start_afflvl < MPIDR_AFFLVL0)
	return PSCI_E_INVALID_PARAMS;

	if (end_afflvl < start_afflvl)
	return PSCI_E_INVALID_PARAMS;

	return PSCI_E_SUCCESS;
	}

	/*******************************************************************************
	* This function is passed an array of pointers to affinity level nodes in the
	* topology tree for an mpidr. It picks up locks for each affinity level bottom
	* up in the range specified.
	******************************************************************************/
	void psci_acquire_afflvl_locks(unsigned long mpidr,
	int start_afflvl,
	int end_afflvl,
	mpidr_aff_map_nodes mpidr_nodes)
	{
	int level;

	for (level = start_afflvl; level <= end_afflvl; level++) {
	if (mpidr_nodes[level] == NULL)
	continue;
	bakery_lock_get(mpidr, &mpidr_nodes[level]->lock);
	}
	}

	/*******************************************************************************
	* This function is passed an array of pointers to affinity level nodes in the
	* topology tree for an mpidr. It releases the lock for each affinity level top
	* down in the range specified.
	******************************************************************************/
	void psci_release_afflvl_locks(unsigned long mpidr,
	int start_afflvl,
	int end_afflvl,
	mpidr_aff_map_nodes mpidr_nodes)
	{
	int level;

	for (level = end_afflvl; level >= start_afflvl; level--) {
	if (mpidr_nodes[level] == NULL)
	continue;
	bakery_lock_release(mpidr, &mpidr_nodes[level]->lock);
	}
	}

	/*******************************************************************************
	* Simple routine to determine whether an affinity instance at a given level
	* in an mpidr exists or not.
	******************************************************************************/
	int psci_validate_mpidr(unsigned long mpidr, int level)
	{
	aff_map_node *node;

	node = psci_get_aff_map_node(mpidr, level);
	if (node && (node->state & PSCI_AFF_PRESENT))
	return PSCI_E_SUCCESS;
	else
	return PSCI_E_INVALID_PARAMS;
	}

	/*******************************************************************************
	* Simple routine to determine the first affinity level instance that is present
	* between the start and end affinity levels. This helps to skip handling of
	* absent affinity levels while performing psci operations.
	* The start level can be > or <= to the end level depending upon whether this
	* routine is expected to search top down or bottom up.
	******************************************************************************/
	int psci_get_first_present_afflvl(unsigned long mpidr,
	int start_afflvl,
	int end_afflvl,
	aff_map_node **node)
	{
	int level;

	/* Check whether we have to search up or down */
	if (start_afflvl <= end_afflvl) {
	for (level = start_afflvl; level <= end_afflvl; level++) {
	*node = psci_get_aff_map_node(mpidr, level);
	if (node && ((node)->state & PSCI_AFF_PRESENT))
	break;
	}
	} else {
	for (level = start_afflvl; level >= end_afflvl; level--) {
	*node = psci_get_aff_map_node(mpidr, level);
	if (node && ((node)->state & PSCI_AFF_PRESENT))
	break;
	}
	}

	return level;
	}

	/*******************************************************************************
	* Iteratively change the affinity state between the current and target affinity
	* levels. The target state matters only if we are starting from affinity level
	* 0 i.e. a cpu otherwise the state depends upon the state of the lower affinity
	* levels.
	******************************************************************************/
	int psci_change_state(mpidr_aff_map_nodes mpidr_nodes,
	int start_afflvl,
	int end_afflvl,
	unsigned int tgt_state)
	{
	int rc = PSCI_E_SUCCESS, level;
	unsigned int state;
	aff_map_node *node;

	/*
	* Get a temp pointer to the node. It is not possible that affinity
	* level 0 is missing. Simply ignore higher missing levels.
	*/
	for (level = start_afflvl; level <= end_afflvl; level++) {

	node = mpidr_nodes[level];
	if (level == MPIDR_AFFLVL0) {
	assert(node);
	psci_set_state(node->state, tgt_state);
	} else {
	if (node == NULL)
	continue;
	state = psci_calculate_affinity_state(node);
	psci_set_state(node->state, state);
	}
	}

	/* If all went well then the cpu should be in the target state */
	if (start_afflvl == MPIDR_AFFLVL0) {
	node = mpidr_nodes[MPIDR_AFFLVL0];
	state = psci_get_state(node->state);
	assert(tgt_state == state);
	}

	return rc;
	}

	/*******************************************************************************
	* This routine does the heavy lifting for psci_change_state(). It examines the
	* state of each affinity instance at the next lower affinity level and decides
	* its final state accordingly. If a lower affinity instance is ON then the
	* higher affinity instance is ON. If all the lower affinity instances are OFF
	* then the higher affinity instance is OFF. If atleast one lower affinity
	* instance is SUSPENDED then the higher affinity instance is SUSPENDED. If only
	* a single lower affinity instance is ON_PENDING then the higher affinity
	* instance in ON_PENDING as well.
	******************************************************************************/
	unsigned int psci_calculate_affinity_state(aff_map_node *aff_node)
	{
	int ctr;
	unsigned int aff_count, hi_aff_state;
	unsigned long tempidr;
	aff_map_node *lo_aff_node;

	/* Cannot calculate lowest affinity state. It is simply assigned */
	assert(aff_node->level > MPIDR_AFFLVL0);

	/*
	* Find the number of affinity instances at level X-1 e.g. number of
	* cpus in a cluster. The level X state depends upon the state of each
	* instance at level X-1
	*/
	hi_aff_state = PSCI_STATE_OFF;
	aff_count = plat_get_aff_count(aff_node->level - 1, aff_node->mpidr);
	for (ctr = 0; ctr < aff_count; ctr++) {

	/*
	* Create a mpidr for each lower affinity level (X-1). Use their
	* states to influence the higher affinity state (X).
	*/
	tempidr = mpidr_set_aff_inst(aff_node->mpidr,
	ctr,
	aff_node->level - 1);
	lo_aff_node = psci_get_aff_map_node(tempidr,
	aff_node->level - 1);
	assert(lo_aff_node);

	/* Continue only if the cpu exists within the cluster */
	if (!(lo_aff_node->state & PSCI_AFF_PRESENT))
	continue;

	switch (psci_get_state(lo_aff_node->state)) {

	/*
	* If any lower affinity is on within the cluster, then
	* the higher affinity is on.
	*/
	case PSCI_STATE_ON:
	return PSCI_STATE_ON;

	/*
	* At least one X-1 needs to be suspended for X to be suspended
	* but it is effectively on for the affinity_info call.
	* SUSPEND > ON_PENDING > OFF.
	*/
	case PSCI_STATE_SUSPEND:
	hi_aff_state = PSCI_STATE_SUSPEND;
	continue;

	/*
	* Atleast one X-1 needs to be on_pending & the rest off for X
	* to be on_pending. ON_PENDING > OFF.
	*/
	case PSCI_STATE_ON_PENDING:
	if (hi_aff_state != PSCI_STATE_SUSPEND)
	hi_aff_state = PSCI_STATE_ON_PENDING;
	continue;

	/* Higher affinity is off if all lower affinities are off. */
	case PSCI_STATE_OFF:
	continue;

	default:
	assert(0);
	}
	}

	return hi_aff_state;
	}

	/*******************************************************************************
	* This function retrieves all the stashed information needed to correctly
	* resume a cpu's execution in the non-secure state after it has been physically
	* powered on i.e. turned ON or resumed from SUSPEND
	******************************************************************************/
	void psci_get_ns_entry_info(unsigned int index)
	{
	unsigned long sctlr = 0, scr, el_status, id_aa64pfr0;
	gp_regs *ns_gp_regs;

	scr = read_scr();

	/* Switch to the non-secure view of the registers */
	write_scr(scr \| SCR_NS_BIT);

	/* Find out which EL we are going to */
	id_aa64pfr0 = read_id_aa64pfr0_el1();
	el_status = (id_aa64pfr0 >> ID_AA64PFR0_EL2_SHIFT) &
	ID_AA64PFR0_ELX_MASK;

	/* Restore endianess */
	if (psci_ns_entry_info[index].sctlr & SCTLR_EE_BIT)
	sctlr \|= SCTLR_EE_BIT;
	else
	sctlr &= ~SCTLR_EE_BIT;

	/* Turn off MMU and Caching */
	sctlr &= ~(SCTLR_M_BIT \| SCTLR_C_BIT \| SCTLR_M_BIT);

	/* Set the register width */
	if (psci_ns_entry_info[index].scr & SCR_RW_BIT)
	scr \|= SCR_RW_BIT;
	else
	scr &= ~SCR_RW_BIT;

	scr \|= SCR_NS_BIT;

	if (el_status)
	write_sctlr_el2(sctlr);
	else
	write_sctlr_el1(sctlr);

	/* Fulfill the cpu_on entry reqs. as per the psci spec */
	write_scr(scr);
	write_elr(psci_ns_entry_info[index].eret_info.entrypoint);

	/*
	* Set the general purpose registers to ~0 upon entry into the
	* non-secure world except for x0 which should contain the
	* context id & spsr. This is done directly on the "would be"
	* stack pointer. Prior to entry into the non-secure world, an
	* offset equivalent to the size of the 'gp_regs' structure is
	* added to the sp. This general purpose register context is
	* retrieved then.
	*/
	ns_gp_regs = (gp_regs *) platform_get_stack(read_mpidr());
	ns_gp_regs--;
	memset(ns_gp_regs, ~0, sizeof(*ns_gp_regs));
	ns_gp_regs->x0 = psci_ns_entry_info[index].context_id;
	ns_gp_regs->spsr = psci_ns_entry_info[index].eret_info.spsr;
	}

	/*******************************************************************************
	* This function retrieves and stashes all the information needed to correctly
	* resume a cpu's execution in the non-secure state after it has been physically
	* powered on i.e. turned ON or resumed from SUSPEND. This is done prior to
	* turning it on or before suspending it.
	******************************************************************************/
	int psci_set_ns_entry_info(unsigned int index,
	unsigned long entrypoint,
	unsigned long context_id)
	{
	int rc = PSCI_E_SUCCESS;
	unsigned int rw, mode, ee, spsr = 0;
	unsigned long id_aa64pfr0 = read_id_aa64pfr0_el1(), scr = read_scr();
	unsigned long el_status;

	/* Figure out what mode do we enter the non-secure world in */
	el_status = (id_aa64pfr0 >> ID_AA64PFR0_EL2_SHIFT) &
	ID_AA64PFR0_ELX_MASK;

	/*
	* Figure out whether the cpu enters the non-secure address space
	* in aarch32 or aarch64
	*/
	rw = scr & SCR_RW_BIT;
	if (rw) {

	/*
	* Check whether a Thumb entry point has been provided for an
	* aarch64 EL
	*/
	if (entrypoint & 0x1)
	return PSCI_E_INVALID_PARAMS;

	if (el_status && (scr & SCR_HCE_BIT)) {
	mode = MODE_EL2;
	ee = read_sctlr_el2() & SCTLR_EE_BIT;
	} else {
	mode = MODE_EL1;
	ee = read_sctlr_el1() & SCTLR_EE_BIT;
	}

	spsr = DAIF_DBG_BIT \| DAIF_ABT_BIT;
	spsr \|= DAIF_IRQ_BIT \| DAIF_FIQ_BIT;
	spsr <<= PSR_DAIF_SHIFT;
	spsr \|= make_spsr(mode, MODE_SP_ELX, !rw);

	psci_ns_entry_info[index].sctlr \|= ee;
	psci_ns_entry_info[index].scr \|= SCR_RW_BIT;
	} else {

	/* Check whether aarch32 has to be entered in Thumb mode */
	if (entrypoint & 0x1)
	spsr = SPSR32_T_BIT;

	if (el_status && (scr & SCR_HCE_BIT)) {
	mode = AARCH32_MODE_HYP;
	ee = read_sctlr_el2() & SCTLR_EE_BIT;
	} else {
	mode = AARCH32_MODE_SVC;
	ee = read_sctlr_el1() & SCTLR_EE_BIT;
	}

	/*
	* TODO: Choose async. exception bits if HYP mode is not
	* implemented according to the values of SCR.{AW, FW} bits
	*/
	spsr \|= DAIF_ABT_BIT \| DAIF_IRQ_BIT \| DAIF_FIQ_BIT;
	spsr <<= PSR_DAIF_SHIFT;
	if(ee)
	spsr \|= SPSR32_EE_BIT;
	spsr \|= mode;

	/* Ensure that the CSPR.E and SCTLR.EE bits match */
	psci_ns_entry_info[index].sctlr \|= ee;
	psci_ns_entry_info[index].scr &= ~SCR_RW_BIT;
	}

	psci_ns_entry_info[index].eret_info.entrypoint = entrypoint;
	psci_ns_entry_info[index].eret_info.spsr = spsr;
	psci_ns_entry_info[index].context_id = context_id;

	return rc;
	}

	/*******************************************************************************
	* An affinity level could be on, on_pending, suspended or off. These are the
	* logical states it can be in. Physically either it is off or on. When it is in
	* the state on_pending then it is about to be turned on. It is not possible to
	* tell whether that's actually happenned or not. So we err on the side of
	* caution & treat the affinity level as being turned off.
	******************************************************************************/
	inline unsigned int psci_get_phys_state(unsigned int aff_state)
	{
	return (aff_state != PSCI_STATE_ON ? PSCI_STATE_OFF : PSCI_STATE_ON);
	}

	unsigned int psci_get_aff_phys_state(aff_map_node *aff_node)
	{
	unsigned int aff_state;

	aff_state = psci_get_state(aff_node->state);
	return psci_get_phys_state(aff_state);
	}

	/*******************************************************************************
	* This function takes an array of pointers to affinity instance nodes in the
	* topology tree and calls the physical power on handler for the corresponding
	* affinity levels
	******************************************************************************/
	static int psci_call_power_on_handlers(mpidr_aff_map_nodes mpidr_nodes,
	int start_afflvl,
	int end_afflvl,
	afflvl_power_on_finisher *pon_handlers,
	unsigned long mpidr)
	{
	int rc = PSCI_E_INVALID_PARAMS, level;
	aff_map_node *node;

	for (level = end_afflvl; level >= start_afflvl; level--) {
	node = mpidr_nodes[level];
	if (node == NULL)
	continue;

	/*
	* If we run into any trouble while powering up an
	* affinity instance, then there is no recovery path
	* so simply return an error and let the caller take
	* care of the situation.
	*/
	rc = pon_handlers[level](mpidr, node);
	if (rc != PSCI_E_SUCCESS)
	break;
	}

	return rc;
	}

	/*******************************************************************************
	* Generic handler which is called when a cpu is physically powered on. It
	* traverses through all the affinity levels performing generic, architectural,
	* platform setup and state management e.g. for a cluster that's been powered
	* on, it will call the platform specific code which will enable coherency at
	* the interconnect level. For a cpu it could mean turning on the MMU etc.
	*
	* The state of all the relevant affinity levels is changed after calling the
	* affinity level specific handlers as their actions would depend upon the state
	* the affinity level is exiting from.
	*
	* The affinity level specific handlers are called in descending order i.e. from
	* the highest to the lowest affinity level implemented by the platform because
	* to turn on affinity level X it is neccesary to turn on affinity level X + 1
	* first.
	*
	* CAUTION: This function is called with coherent stacks so that coherency and
	* the mmu can be turned on safely.
	******************************************************************************/
	void psci_afflvl_power_on_finish(unsigned long mpidr,
	int start_afflvl,
	int end_afflvl,
	afflvl_power_on_finisher *pon_handlers)
	{
	mpidr_aff_map_nodes mpidr_nodes;
	int rc;

	mpidr &= MPIDR_AFFINITY_MASK;;

	/*
	* Collect the pointers to the nodes in the topology tree for
	* each affinity instance in the mpidr. If this function does
	* not return successfully then either the mpidr or the affinity
	* levels are incorrect. Either case is an irrecoverable error.
	*/
	rc = psci_get_aff_map_nodes(mpidr,
	start_afflvl,
	end_afflvl,
	mpidr_nodes);
	assert (rc == PSCI_E_SUCCESS);

	/*
	* This function acquires the lock corresponding to each affinity
	* level so that by the time all locks are taken, the system topology
	* is snapshot and state management can be done safely.
	*/
	psci_acquire_afflvl_locks(mpidr,
	start_afflvl,
	end_afflvl,
	mpidr_nodes);

	/* Perform generic, architecture and platform specific handling */
	rc = psci_call_power_on_handlers(mpidr_nodes,
	start_afflvl,
	end_afflvl,
	pon_handlers,
	mpidr);
	assert (rc == PSCI_E_SUCCESS);

	/*
	* State management: Update the state of each affinity instance
	* between the start and end affinity levels
	*/
	psci_change_state(mpidr_nodes,
	start_afflvl,
	end_afflvl,
	PSCI_STATE_ON);

	/*
	* This loop releases the lock corresponding to each affinity level
	* in the reverse order to which they were acquired.
	*/
	psci_release_afflvl_locks(mpidr,
	start_afflvl,
	end_afflvl,
	mpidr_nodes);
	}