arch/arm/common/mcpm_entry.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
  *
  * Created by:  Nicolas Pitre, March 2012
  * Copyright:   (C) 2012-2013  Linaro Limited
  */

 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/irqflags.h>
 #include <linux/cpu_pm.h>

 #include <asm/mcpm.h>
 #include <asm/cacheflush.h>
 #include <asm/idmap.h>
 #include <asm/cputype.h>
 #include <asm/suspend.h>

 /*
  * The public API for this code is documented in arch/arm/include/asm/mcpm.h.
  * For a comprehensive description of the main algorithm used here, please
  * see Documentation/arm/cluster-pm-race-avoidance.rst.
  */

 struct sync_struct mcpm_sync;

 /*
  * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
  *    This must be called at the point of committing to teardown of a CPU.
  *    The CPU cache (SCTRL.C bit) is expected to still be active.
  */
 static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
 {
 	mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
 	sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
 }

 /*
  * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
  *    cluster can be torn down without disrupting this CPU.
  *    To avoid deadlocks, this must be called before a CPU is powered down.
  *    The CPU cache (SCTRL.C bit) is expected to be off.
  *    However L2 cache might or might not be active.
  */
 static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
 {
 	dmb();
 	mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
 	sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
 	sev();
 }

 /*
  * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
  * @state: the final state of the cluster:
  *     CLUSTER_UP: no destructive teardown was done and the cluster has been
  *         restored to the previous state (CPU cache still active); or
  *     CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
  *         (CPU cache disabled, L2 cache either enabled or disabled).
  */
 static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
 {
 	dmb();
 	mcpm_sync.clusters[cluster].cluster = state;
 	sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
 	sev();
 }

 /*
  * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
  * This function should be called by the last man, after local CPU teardown
  * is complete.  CPU cache expected to be active.
  *
  * Returns:
  *     false: the critical section was not entered because an inbound CPU was
  *         observed, or the cluster is already being set up;
  *     true: the critical section was entered: it is now safe to tear down the
  *         cluster.
  */
 static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
 {
 	unsigned int i;
 	struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];

 	/* Warn inbound CPUs that the cluster is being torn down: */
 	c->cluster = CLUSTER_GOING_DOWN;
 	sync_cache_w(&c->cluster);

 	/* Back out if the inbound cluster is already in the critical region: */
 	sync_cache_r(&c->inbound);
 	if (c->inbound == INBOUND_COMING_UP)
 		goto abort;

 	/*
 	 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
 	 * teardown is complete on each CPU before tearing down the cluster.
 	 *
 	 * If any CPU has been woken up again from the DOWN state, then we
 	 * shouldn't be taking the cluster down at all: abort in that case.
 	 */
 	sync_cache_r(&c->cpus);
 	for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
 		int cpustate;

 		if (i == cpu)
 			continue;

 		while (1) {
 			cpustate = c->cpus[i].cpu;
 			if (cpustate != CPU_GOING_DOWN)
 				break;

 			wfe();
 			sync_cache_r(&c->cpus[i].cpu);
 		}

 		switch (cpustate) {
 		case CPU_DOWN:
 			continue;

 		default:
 			goto abort;
 		}
 	}

 	return true;

 abort:
 	__mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
 	return false;
 }

 static int __mcpm_cluster_state(unsigned int cluster)
 {
 	sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
 	return mcpm_sync.clusters[cluster].cluster;
 }

 extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];

 void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
 {
 	unsigned long val = ptr ? __pa_symbol(ptr) : 0;
 	mcpm_entry_vectors[cluster][cpu] = val;
 	sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
 }

 extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];

 void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
 			 unsigned long poke_phys_addr, unsigned long poke_val)
 {
 	unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
 	poke[0] = poke_phys_addr;
 	poke[1] = poke_val;
 	__sync_cache_range_w(poke, 2 * sizeof(*poke));
 }

 static const struct mcpm_platform_ops *platform_ops;

 int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
 {
 	if (platform_ops)
 		return -EBUSY;
 	platform_ops = ops;
 	return 0;
 }

 bool mcpm_is_available(void)
 {
 	return (platform_ops) ? true : false;
 }
 EXPORT_SYMBOL_GPL(mcpm_is_available);

 /*
  * We can't use regular spinlocks. In the switcher case, it is possible
  * for an outbound CPU to call power_down() after its inbound counterpart
  * is already live using the same logical CPU number which trips lockdep
  * debugging.
  */
 static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;

 static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];

 static inline bool mcpm_cluster_unused(unsigned int cluster)
 {
 	int i, cnt;
 	for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
 		cnt |= mcpm_cpu_use_count[cluster][i];
 	return !cnt;
 }

 int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
 {
 	bool cpu_is_down, cluster_is_down;
 	int ret = 0;

 	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
 	if (!platform_ops)
 		return -EUNATCH; /* try not to shadow power_up errors */
 	might_sleep();

 	/*
 	 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
 	 * variant exists, we need to disable IRQs manually here.
 	 */
 	local_irq_disable();
 	arch_spin_lock(&mcpm_lock);

 	cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
 	cluster_is_down = mcpm_cluster_unused(cluster);

 	mcpm_cpu_use_count[cluster][cpu]++;
 	/*
 	 * The only possible values are:
 	 * 0 = CPU down
 	 * 1 = CPU (still) up
 	 * 2 = CPU requested to be up before it had a chance
 	 *     to actually make itself down.
 	 * Any other value is a bug.
 	 */
 	BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
 	       mcpm_cpu_use_count[cluster][cpu] != 2);

 	if (cluster_is_down)
 		ret = platform_ops->cluster_powerup(cluster);
 	if (cpu_is_down && !ret)
 		ret = platform_ops->cpu_powerup(cpu, cluster);

 	arch_spin_unlock(&mcpm_lock);
 	local_irq_enable();
 	return ret;
 }

 typedef typeof(cpu_reset) phys_reset_t;

 void mcpm_cpu_power_down(void)
 {
 	unsigned int mpidr, cpu, cluster;
 	bool cpu_going_down, last_man;
 	phys_reset_t phys_reset;

 	mpidr = read_cpuid_mpidr();
 	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
 	if (WARN_ON_ONCE(!platform_ops))
 	       return;
 	BUG_ON(!irqs_disabled());

 	setup_mm_for_reboot();

 	__mcpm_cpu_going_down(cpu, cluster);
 	arch_spin_lock(&mcpm_lock);
 	BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);

 	mcpm_cpu_use_count[cluster][cpu]--;
 	BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
 	       mcpm_cpu_use_count[cluster][cpu] != 1);
 	cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
 	last_man = mcpm_cluster_unused(cluster);

 	if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
 		platform_ops->cpu_powerdown_prepare(cpu, cluster);
 		platform_ops->cluster_powerdown_prepare(cluster);
 		arch_spin_unlock(&mcpm_lock);
 		platform_ops->cluster_cache_disable();
 		__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
 	} else {
 		if (cpu_going_down)
 			platform_ops->cpu_powerdown_prepare(cpu, cluster);
 		arch_spin_unlock(&mcpm_lock);
 		/*
 		 * If cpu_going_down is false here, that means a power_up
 		 * request raced ahead of us.  Even if we do not want to
 		 * shut this CPU down, the caller still expects execution
 		 * to return through the system resume entry path, like
 		 * when the WFI is aborted due to a new IRQ or the like..
 		 * So let's continue with cache cleaning in all cases.
 		 */
 		platform_ops->cpu_cache_disable();
 	}

 	__mcpm_cpu_down(cpu, cluster);

 	/* Now we are prepared for power-down, do it: */
 	if (cpu_going_down)
 		wfi();

 	/*
 	 * It is possible for a power_up request to happen concurrently
 	 * with a power_down request for the same CPU. In this case the
 	 * CPU might not be able to actually enter a powered down state
 	 * with the WFI instruction if the power_up request has removed
 	 * the required reset condition.  We must perform a re-entry in
 	 * the kernel as if the power_up method just had deasserted reset
 	 * on the CPU.
 	 */
 	phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
 	phys_reset(__pa_symbol(mcpm_entry_point), false);

 	/* should never get here */
 	BUG();
 }

 int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
 {
 	int ret;

 	if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
 		return -EUNATCH;

 	ret = platform_ops->wait_for_powerdown(cpu, cluster);
 	if (ret)
 		pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
 			__func__, cpu, cluster, ret);

 	return ret;
 }

 void mcpm_cpu_suspend(void)
 {
 	if (WARN_ON_ONCE(!platform_ops))
 		return;

 	/* Some platforms might have to enable special resume modes, etc. */
 	if (platform_ops->cpu_suspend_prepare) {
 		unsigned int mpidr = read_cpuid_mpidr();
 		unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 		unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 		arch_spin_lock(&mcpm_lock);
 		platform_ops->cpu_suspend_prepare(cpu, cluster);
 		arch_spin_unlock(&mcpm_lock);
 	}
 	mcpm_cpu_power_down();
 }

 int mcpm_cpu_powered_up(void)
 {
 	unsigned int mpidr, cpu, cluster;
 	bool cpu_was_down, first_man;
 	unsigned long flags;

 	if (!platform_ops)
 		return -EUNATCH;

 	mpidr = read_cpuid_mpidr();
 	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 	local_irq_save(flags);
 	arch_spin_lock(&mcpm_lock);

 	cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
 	first_man = mcpm_cluster_unused(cluster);

 	if (first_man && platform_ops->cluster_is_up)
 		platform_ops->cluster_is_up(cluster);
 	if (cpu_was_down)
 		mcpm_cpu_use_count[cluster][cpu] = 1;
 	if (platform_ops->cpu_is_up)
 		platform_ops->cpu_is_up(cpu, cluster);

 	arch_spin_unlock(&mcpm_lock);
 	local_irq_restore(flags);

 	return 0;
 }

 #ifdef CONFIG_ARM_CPU_SUSPEND

 static int __init nocache_trampoline(unsigned long _arg)
 {
 	void (*cache_disable)(void) = (void *)_arg;
 	unsigned int mpidr = read_cpuid_mpidr();
 	unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 	unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 	phys_reset_t phys_reset;

 	mcpm_set_entry_vector(cpu, cluster, cpu_resume_no_hyp);
 	setup_mm_for_reboot();

 	__mcpm_cpu_going_down(cpu, cluster);
 	BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
 	cache_disable();
 	__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
 	__mcpm_cpu_down(cpu, cluster);

 	phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
 	phys_reset(__pa_symbol(mcpm_entry_point), false);
 	BUG();
 }

 int __init mcpm_loopback(void (*cache_disable)(void))
 {
 	int ret;

 	/*
 	 * We're going to soft-restart the current CPU through the
 	 * low-level MCPM code by leveraging the suspend/resume
 	 * infrastructure. Let's play it safe by using cpu_pm_enter()
 	 * in case the CPU init code path resets the VFP or similar.
 	 */
 	local_irq_disable();
 	local_fiq_disable();
 	ret = cpu_pm_enter();
 	if (!ret) {
 		ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
 		cpu_pm_exit();
 	}
 	local_fiq_enable();
 	local_irq_enable();
 	if (ret)
 		pr_err("%s returned %d\n", __func__, ret);
 	return ret;
 }

 #endif

 extern unsigned long mcpm_power_up_setup_phys;

 int __init mcpm_sync_init(
 	void (*power_up_setup)(unsigned int affinity_level))
 {
 	unsigned int i, j, mpidr, this_cluster;

 	BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
 	BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));

 	/*
 	 * Set initial CPU and cluster states.
 	 * Only one cluster is assumed to be active at this point.
 	 */
 	for (i = 0; i < MAX_NR_CLUSTERS; i++) {
 		mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
 		mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
 		for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
 			mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
 	}
 	mpidr = read_cpuid_mpidr();
 	this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 	for_each_online_cpu(i) {
 		mcpm_cpu_use_count[this_cluster][i] = 1;
 		mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
 	}
 	mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
 	sync_cache_w(&mcpm_sync);

 	if (power_up_setup) {
 		mcpm_power_up_setup_phys = __pa_symbol(power_up_setup);
 		sync_cache_w(&mcpm_power_up_setup_phys);
 	}

 	return 0;
 }
	// SPDX-License-Identifier: GPL-2.0-only
	/*
	* arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
	*
	* Created by: Nicolas Pitre, March 2012
	* Copyright: (C) 2012-2013 Linaro Limited
	*/

	#include <linux/export.h>
	#include <linux/kernel.h>
	#include <linux/init.h>
	#include <linux/irqflags.h>
	#include <linux/cpu_pm.h>

	#include <asm/mcpm.h>
	#include <asm/cacheflush.h>
	#include <asm/idmap.h>
	#include <asm/cputype.h>
	#include <asm/suspend.h>

	/*
	* The public API for this code is documented in arch/arm/include/asm/mcpm.h.
	* For a comprehensive description of the main algorithm used here, please
	* see Documentation/arm/cluster-pm-race-avoidance.rst.
	*/

	struct sync_struct mcpm_sync;

	/*
	* __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
	* This must be called at the point of committing to teardown of a CPU.
	* The CPU cache (SCTRL.C bit) is expected to still be active.
	*/
	static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
	{
	mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
	sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
	}

	/*
	* __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
	* cluster can be torn down without disrupting this CPU.
	* To avoid deadlocks, this must be called before a CPU is powered down.
	* The CPU cache (SCTRL.C bit) is expected to be off.
	* However L2 cache might or might not be active.
	*/
	static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
	{
	dmb();
	mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
	sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
	sev();
	}

	/*
	* __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
	* @state: the final state of the cluster:
	* CLUSTER_UP: no destructive teardown was done and the cluster has been
	* restored to the previous state (CPU cache still active); or
	* CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
	* (CPU cache disabled, L2 cache either enabled or disabled).
	*/
	static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
	{
	dmb();
	mcpm_sync.clusters[cluster].cluster = state;
	sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
	sev();
	}

	/*
	* __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
	* This function should be called by the last man, after local CPU teardown
	* is complete. CPU cache expected to be active.
	*
	* Returns:
	* false: the critical section was not entered because an inbound CPU was
	* observed, or the cluster is already being set up;
	* true: the critical section was entered: it is now safe to tear down the
	* cluster.
	*/
	static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
	{
	unsigned int i;
	struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];

	/* Warn inbound CPUs that the cluster is being torn down: */
	c->cluster = CLUSTER_GOING_DOWN;
	sync_cache_w(&c->cluster);

	/* Back out if the inbound cluster is already in the critical region: */
	sync_cache_r(&c->inbound);
	if (c->inbound == INBOUND_COMING_UP)
	goto abort;

	/*
	* Wait for all CPUs to get out of the GOING_DOWN state, so that local
	* teardown is complete on each CPU before tearing down the cluster.
	*
	* If any CPU has been woken up again from the DOWN state, then we
	* shouldn't be taking the cluster down at all: abort in that case.
	*/
	sync_cache_r(&c->cpus);
	for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
	int cpustate;

	if (i == cpu)
	continue;

	while (1) {
	cpustate = c->cpus[i].cpu;
	if (cpustate != CPU_GOING_DOWN)
	break;

	wfe();
	sync_cache_r(&c->cpus[i].cpu);
	}

	switch (cpustate) {
	case CPU_DOWN:
	continue;

	default:
	goto abort;
	}
	}

	return true;

	abort:
	__mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
	return false;
	}

	static int __mcpm_cluster_state(unsigned int cluster)
	{
	sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
	return mcpm_sync.clusters[cluster].cluster;
	}

	extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];

	void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
	{
	unsigned long val = ptr ? __pa_symbol(ptr) : 0;
	mcpm_entry_vectors[cluster][cpu] = val;
	sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
	}

	extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];

	void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
	unsigned long poke_phys_addr, unsigned long poke_val)
	{
	unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
	poke[0] = poke_phys_addr;
	poke[1] = poke_val;
	__sync_cache_range_w(poke, 2 * sizeof(*poke));
	}

	static const struct mcpm_platform_ops *platform_ops;

	int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
	{
	if (platform_ops)
	return -EBUSY;
	platform_ops = ops;
	return 0;
	}

	bool mcpm_is_available(void)
	{
	return (platform_ops) ? true : false;
	}
	EXPORT_SYMBOL_GPL(mcpm_is_available);

	/*
	* We can't use regular spinlocks. In the switcher case, it is possible
	* for an outbound CPU to call power_down() after its inbound counterpart
	* is already live using the same logical CPU number which trips lockdep
	* debugging.
	*/
	static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;

	static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];

	static inline bool mcpm_cluster_unused(unsigned int cluster)
	{
	int i, cnt;
	for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
	cnt \|= mcpm_cpu_use_count[cluster][i];
	return !cnt;
	}

	int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
	{
	bool cpu_is_down, cluster_is_down;
	int ret = 0;

	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
	if (!platform_ops)
	return -EUNATCH; /* try not to shadow power_up errors */
	might_sleep();

	/*
	* Since this is called with IRQs enabled, and no arch_spin_lock_irq
	* variant exists, we need to disable IRQs manually here.
	*/
	local_irq_disable();
	arch_spin_lock(&mcpm_lock);

	cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
	cluster_is_down = mcpm_cluster_unused(cluster);

	mcpm_cpu_use_count[cluster][cpu]++;
	/*
	* The only possible values are:
	* 0 = CPU down
	* 1 = CPU (still) up
	* 2 = CPU requested to be up before it had a chance
	* to actually make itself down.
	* Any other value is a bug.
	*/
	BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
	mcpm_cpu_use_count[cluster][cpu] != 2);

	if (cluster_is_down)
	ret = platform_ops->cluster_powerup(cluster);
	if (cpu_is_down && !ret)
	ret = platform_ops->cpu_powerup(cpu, cluster);

	arch_spin_unlock(&mcpm_lock);
	local_irq_enable();
	return ret;
	}

	typedef typeof(cpu_reset) phys_reset_t;

	void mcpm_cpu_power_down(void)
	{
	unsigned int mpidr, cpu, cluster;
	bool cpu_going_down, last_man;
	phys_reset_t phys_reset;

	mpidr = read_cpuid_mpidr();
	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
	if (WARN_ON_ONCE(!platform_ops))
	return;
	BUG_ON(!irqs_disabled());

	setup_mm_for_reboot();

	__mcpm_cpu_going_down(cpu, cluster);
	arch_spin_lock(&mcpm_lock);
	BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);

	mcpm_cpu_use_count[cluster][cpu]--;
	BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
	mcpm_cpu_use_count[cluster][cpu] != 1);
	cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
	last_man = mcpm_cluster_unused(cluster);

	if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
	platform_ops->cpu_powerdown_prepare(cpu, cluster);
	platform_ops->cluster_powerdown_prepare(cluster);
	arch_spin_unlock(&mcpm_lock);
	platform_ops->cluster_cache_disable();
	__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
	} else {
	if (cpu_going_down)
	platform_ops->cpu_powerdown_prepare(cpu, cluster);
	arch_spin_unlock(&mcpm_lock);
	/*
	* If cpu_going_down is false here, that means a power_up
	* request raced ahead of us. Even if we do not want to
	* shut this CPU down, the caller still expects execution
	* to return through the system resume entry path, like
	* when the WFI is aborted due to a new IRQ or the like..
	* So let's continue with cache cleaning in all cases.
	*/
	platform_ops->cpu_cache_disable();
	}

	__mcpm_cpu_down(cpu, cluster);

	/* Now we are prepared for power-down, do it: */
	if (cpu_going_down)
	wfi();

	/*
	* It is possible for a power_up request to happen concurrently
	* with a power_down request for the same CPU. In this case the
	* CPU might not be able to actually enter a powered down state
	* with the WFI instruction if the power_up request has removed
	* the required reset condition. We must perform a re-entry in
	* the kernel as if the power_up method just had deasserted reset
	* on the CPU.
	*/
	phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
	phys_reset(__pa_symbol(mcpm_entry_point), false);

	/* should never get here */
	BUG();
	}

	int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
	{
	int ret;

	if (WARN_ON_ONCE(!platform_ops \|\| !platform_ops->wait_for_powerdown))
	return -EUNATCH;

	ret = platform_ops->wait_for_powerdown(cpu, cluster);
	if (ret)
	pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
	__func__, cpu, cluster, ret);

	return ret;
	}

	void mcpm_cpu_suspend(void)
	{
	if (WARN_ON_ONCE(!platform_ops))
	return;

	/* Some platforms might have to enable special resume modes, etc. */
	if (platform_ops->cpu_suspend_prepare) {
	unsigned int mpidr = read_cpuid_mpidr();
	unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
	unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	arch_spin_lock(&mcpm_lock);
	platform_ops->cpu_suspend_prepare(cpu, cluster);
	arch_spin_unlock(&mcpm_lock);
	}
	mcpm_cpu_power_down();
	}

	int mcpm_cpu_powered_up(void)
	{
	unsigned int mpidr, cpu, cluster;
	bool cpu_was_down, first_man;
	unsigned long flags;

	if (!platform_ops)
	return -EUNATCH;

	mpidr = read_cpuid_mpidr();
	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	local_irq_save(flags);
	arch_spin_lock(&mcpm_lock);

	cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
	first_man = mcpm_cluster_unused(cluster);

	if (first_man && platform_ops->cluster_is_up)
	platform_ops->cluster_is_up(cluster);
	if (cpu_was_down)
	mcpm_cpu_use_count[cluster][cpu] = 1;
	if (platform_ops->cpu_is_up)
	platform_ops->cpu_is_up(cpu, cluster);

	arch_spin_unlock(&mcpm_lock);
	local_irq_restore(flags);

	return 0;
	}

	#ifdef CONFIG_ARM_CPU_SUSPEND

	static int __init nocache_trampoline(unsigned long _arg)
	{
	void (cache_disable)(void) = (void )_arg;
	unsigned int mpidr = read_cpuid_mpidr();
	unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
	unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	phys_reset_t phys_reset;

	mcpm_set_entry_vector(cpu, cluster, cpu_resume_no_hyp);
	setup_mm_for_reboot();

	__mcpm_cpu_going_down(cpu, cluster);
	BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
	cache_disable();
	__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
	__mcpm_cpu_down(cpu, cluster);

	phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
	phys_reset(__pa_symbol(mcpm_entry_point), false);
	BUG();
	}

	int __init mcpm_loopback(void (*cache_disable)(void))
	{
	int ret;

	/*
	* We're going to soft-restart the current CPU through the
	* low-level MCPM code by leveraging the suspend/resume
	* infrastructure. Let's play it safe by using cpu_pm_enter()
	* in case the CPU init code path resets the VFP or similar.
	*/
	local_irq_disable();
	local_fiq_disable();
	ret = cpu_pm_enter();
	if (!ret) {
	ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
	cpu_pm_exit();
	}
	local_fiq_enable();
	local_irq_enable();
	if (ret)
	pr_err("%s returned %d\n", __func__, ret);
	return ret;
	}

	#endif

	extern unsigned long mcpm_power_up_setup_phys;

	int __init mcpm_sync_init(
	void (*power_up_setup)(unsigned int affinity_level))
	{
	unsigned int i, j, mpidr, this_cluster;

	BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
	BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));

	/*
	* Set initial CPU and cluster states.
	* Only one cluster is assumed to be active at this point.
	*/
	for (i = 0; i < MAX_NR_CLUSTERS; i++) {
	mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
	mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
	for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
	mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
	}
	mpidr = read_cpuid_mpidr();
	this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	for_each_online_cpu(i) {
	mcpm_cpu_use_count[this_cluster][i] = 1;
	mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
	}
	mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
	sync_cache_w(&mcpm_sync);

	if (power_up_setup) {
	mcpm_power_up_setup_phys = __pa_symbol(power_up_setup);
	sync_cache_w(&mcpm_power_up_setup_phys);
	}

	return 0;
	}