drivers/powercap/idle_inject.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2018 Linaro Limited
  *
  * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
  *
  * The idle injection framework provides a way to force CPUs to enter idle
  * states for a specified fraction of time over a specified period.
  *
  * It relies on the smpboot kthreads feature providing common code for CPU
  * hotplug and thread [un]parking.
  *
  * All of the kthreads used for idle injection are created at init time.
  *
  * Next, the users of the idle injection framework provide a cpumask via
  * its register function. The kthreads will be synchronized with respect to
  * this cpumask.
  *
  * The idle + run duration is specified via separate helpers and that allows
  * idle injection to be started.
  *
  * The idle injection kthreads will call play_idle_precise() with the idle
  * duration and max allowed latency specified as per the above.
  *
  * After all of them have been woken up, a timer is set to start the next idle
  * injection cycle.
  *
  * The timer interrupt handler will wake up the idle injection kthreads for
  * all of the CPUs in the cpumask provided by the user.
  *
  * Idle injection is stopped synchronously and no leftover idle injection
  * kthread activity after its completion is guaranteed.
  *
  * It is up to the user of this framework to provide a lock for higher-level
  * synchronization to prevent race conditions like starting idle injection
  * while unregistering from the framework.
  */
 #define pr_fmt(fmt) "ii_dev: " fmt

 #include <linux/cpu.h>
 #include <linux/hrtimer.h>
 #include <linux/kthread.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/smpboot.h>
 #include <linux/idle_inject.h>

 #include <uapi/linux/sched/types.h>

 /**
  * struct idle_inject_thread - task on/off switch structure
  * @tsk: task injecting the idle cycles
  * @should_run: whether or not to run the task (for the smpboot kthread API)
  */
 struct idle_inject_thread {
 	struct task_struct *tsk;
 	int should_run;
 };

 /**
  * struct idle_inject_device - idle injection data
  * @timer: idle injection period timer
  * @idle_duration_us: duration of CPU idle time to inject
  * @run_duration_us: duration of CPU run time to allow
  * @latency_us: max allowed latency
  * @update: Optional callback deciding whether or not to skip idle
  *		injection in the given cycle.
  * @cpumask: mask of CPUs affected by idle injection
  *
  * This structure is used to define per instance idle inject device data. Each
  * instance has an idle duration, a run duration and mask of CPUs to inject
  * idle.
  *
  * Actual CPU idle time is injected by calling kernel scheduler interface
  * play_idle_precise(). There is one optional callback that can be registered
  * by calling idle_inject_register_full():
  *
  * update() - This callback is invoked just before waking up CPUs to inject
  * idle. If it returns false, CPUs are not woken up to inject idle in the given
  * cycle. It also allows the caller to readjust the idle and run duration by
  * calling idle_inject_set_duration() for the next cycle.
  */
 struct idle_inject_device {
 	struct hrtimer timer;
 	unsigned int idle_duration_us;
 	unsigned int run_duration_us;
 	unsigned int latency_us;
 	bool (*update)(void);
 	unsigned long cpumask[];
 };

 static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
 static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);

 /**
  * idle_inject_wakeup - Wake up idle injection threads
  * @ii_dev: target idle injection device
  *
  * Every idle injection task associated with the given idle injection device
  * and running on an online CPU will be woken up.
  */
 static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
 {
 	struct idle_inject_thread *iit;
 	unsigned int cpu;

 	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
 		iit = per_cpu_ptr(&idle_inject_thread, cpu);
 		iit->should_run = 1;
 		wake_up_process(iit->tsk);
 	}
 }

 /**
  * idle_inject_timer_fn - idle injection timer function
  * @timer: idle injection hrtimer
  *
  * This function is called when the idle injection timer expires.  It wakes up
  * idle injection tasks associated with the timer and they, in turn, invoke
  * play_idle_precise() to inject a specified amount of CPU idle time.
  *
  * Return: HRTIMER_RESTART.
  */
 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
 {
 	unsigned int duration_us;
 	struct idle_inject_device *ii_dev =
 		container_of(timer, struct idle_inject_device, timer);

 	if (!ii_dev->update || (ii_dev->update && ii_dev->update()))
 		idle_inject_wakeup(ii_dev);

 	duration_us = READ_ONCE(ii_dev->run_duration_us);
 	duration_us += READ_ONCE(ii_dev->idle_duration_us);

 	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));

 	return HRTIMER_RESTART;
 }

 /**
  * idle_inject_fn - idle injection work function
  * @cpu: the CPU owning the task
  *
  * This function calls play_idle_precise() to inject a specified amount of CPU
  * idle time.
  */
 static void idle_inject_fn(unsigned int cpu)
 {
 	struct idle_inject_device *ii_dev;
 	struct idle_inject_thread *iit;

 	ii_dev = per_cpu(idle_inject_device, cpu);
 	iit = per_cpu_ptr(&idle_inject_thread, cpu);

 	/*
 	 * Let the smpboot main loop know that the task should not run again.
 	 */
 	iit->should_run = 0;

 	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
 			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
 }

 /**
  * idle_inject_set_duration - idle and run duration update helper
  * @ii_dev: idle injection control device structure
  * @run_duration_us: CPU run time to allow in microseconds
  * @idle_duration_us: CPU idle time to inject in microseconds
  */
 void idle_inject_set_duration(struct idle_inject_device *ii_dev,
 			      unsigned int run_duration_us,
 			      unsigned int idle_duration_us)
 {
 	if (run_duration_us + idle_duration_us) {
 		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
 		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
 	}
 	if (!run_duration_us)
 		pr_debug("CPU is forced to 100 percent idle\n");
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);

 /**
  * idle_inject_get_duration - idle and run duration retrieval helper
  * @ii_dev: idle injection control device structure
  * @run_duration_us: memory location to store the current CPU run time
  * @idle_duration_us: memory location to store the current CPU idle time
  */
 void idle_inject_get_duration(struct idle_inject_device *ii_dev,
 			      unsigned int *run_duration_us,
 			      unsigned int *idle_duration_us)
 {
 	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
 	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);

 /**
  * idle_inject_set_latency - set the maximum latency allowed
  * @ii_dev: idle injection control device structure
  * @latency_us: set the latency requirement for the idle state
  */
 void idle_inject_set_latency(struct idle_inject_device *ii_dev,
 			     unsigned int latency_us)
 {
 	WRITE_ONCE(ii_dev->latency_us, latency_us);
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);

 /**
  * idle_inject_start - start idle injections
  * @ii_dev: idle injection control device structure
  *
  * The function starts idle injection by first waking up all of the idle
  * injection kthreads associated with @ii_dev to let them inject CPU idle time
  * sets up a timer to start the next idle injection period.
  *
  * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
  */
 int idle_inject_start(struct idle_inject_device *ii_dev)
 {
 	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
 	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);

 	if (!(idle_duration_us + run_duration_us))
 		return -EINVAL;

 	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

 	idle_inject_wakeup(ii_dev);

 	hrtimer_start(&ii_dev->timer,
 		      ns_to_ktime((idle_duration_us + run_duration_us) *
 				  NSEC_PER_USEC),
 		      HRTIMER_MODE_REL);

 	return 0;
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);

 /**
  * idle_inject_stop - stops idle injections
  * @ii_dev: idle injection control device structure
  *
  * The function stops idle injection and waits for the threads to finish work.
  * If CPU idle time is being injected when this function runs, then it will
  * wait until the end of the cycle.
  *
  * When it returns, there is no more idle injection kthread activity.  The
  * kthreads are scheduled out and the periodic timer is off.
  */
 void idle_inject_stop(struct idle_inject_device *ii_dev)
 {
 	struct idle_inject_thread *iit;
 	unsigned int cpu;

 	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

 	hrtimer_cancel(&ii_dev->timer);

 	/*
 	 * Stopping idle injection requires all of the idle injection kthreads
 	 * associated with the given cpumask to be parked and stay that way, so
 	 * prevent CPUs from going online at this point.  Any CPUs going online
 	 * after the loop below will be covered by clearing the should_run flag
 	 * that will cause the smpboot main loop to schedule them out.
 	 */
 	cpu_hotplug_disable();

 	/*
 	 * Iterate over all (online + offline) CPUs here in case one of them
 	 * goes offline with the should_run flag set so as to prevent its idle
 	 * injection kthread from running when the CPU goes online again after
 	 * the ii_dev has been freed.
 	 */
 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
 		iit = per_cpu_ptr(&idle_inject_thread, cpu);
 		iit->should_run = 0;

 		wait_task_inactive(iit->tsk, TASK_ANY);
 	}

 	cpu_hotplug_enable();
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);

 /**
  * idle_inject_setup - prepare the current task for idle injection
  * @cpu: not used
  *
  * Called once, this function is in charge of setting the current task's
  * scheduler parameters to make it an RT task.
  */
 static void idle_inject_setup(unsigned int cpu)
 {
 	sched_set_fifo(current);
 }

 /**
  * idle_inject_should_run - function helper for the smpboot API
  * @cpu: CPU the kthread is running on
  *
  * Return: whether or not the thread can run.
  */
 static int idle_inject_should_run(unsigned int cpu)
 {
 	struct idle_inject_thread *iit =
 		per_cpu_ptr(&idle_inject_thread, cpu);

 	return iit->should_run;
 }

 /**
  * idle_inject_register_full - initialize idle injection on a set of CPUs
  * @cpumask: CPUs to be affected by idle injection
  * @update: This callback is called just before waking up CPUs to inject
  * idle
  *
  * This function creates an idle injection control device structure for the
  * given set of CPUs and initializes the timer associated with it. This
  * function also allows to register update()callback.
  * It does not start any injection cycles.
  *
  * Return: NULL if memory allocation fails, idle injection control device
  * pointer on success.
  */

 struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
 						     bool (*update)(void))
 {
 	struct idle_inject_device *ii_dev;
 	int cpu, cpu_rb;

 	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
 	if (!ii_dev)
 		return NULL;

 	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
 	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	ii_dev->timer.function = idle_inject_timer_fn;
 	ii_dev->latency_us = UINT_MAX;
 	ii_dev->update = update;

 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {

 		if (per_cpu(idle_inject_device, cpu)) {
 			pr_err("cpu%d is already registered\n", cpu);
 			goto out_rollback;
 		}

 		per_cpu(idle_inject_device, cpu) = ii_dev;
 	}

 	return ii_dev;

 out_rollback:
 	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
 		if (cpu == cpu_rb)
 			break;
 		per_cpu(idle_inject_device, cpu_rb) = NULL;
 	}

 	kfree(ii_dev);

 	return NULL;
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);

 /**
  * idle_inject_register - initialize idle injection on a set of CPUs
  * @cpumask: CPUs to be affected by idle injection
  *
  * This function creates an idle injection control device structure for the
  * given set of CPUs and initializes the timer associated with it.  It does not
  * start any injection cycles.
  *
  * Return: NULL if memory allocation fails, idle injection control device
  * pointer on success.
  */
 struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
 {
 	return idle_inject_register_full(cpumask, NULL);
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);

 /**
  * idle_inject_unregister - unregister idle injection control device
  * @ii_dev: idle injection control device to unregister
  *
  * The function stops idle injection for the given control device,
  * unregisters its kthreads and frees memory allocated when that device was
  * created.
  */
 void idle_inject_unregister(struct idle_inject_device *ii_dev)
 {
 	unsigned int cpu;

 	idle_inject_stop(ii_dev);

 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
 		per_cpu(idle_inject_device, cpu) = NULL;

 	kfree(ii_dev);
 }
 EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);

 static struct smp_hotplug_thread idle_inject_threads = {
 	.store = &idle_inject_thread.tsk,
 	.setup = idle_inject_setup,
 	.thread_fn = idle_inject_fn,
 	.thread_comm = "idle_inject/%u",
 	.thread_should_run = idle_inject_should_run,
 };

 static int __init idle_inject_init(void)
 {
 	return smpboot_register_percpu_thread(&idle_inject_threads);
 }
 early_initcall(idle_inject_init);
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright 2018 Linaro Limited
	*
	* Author: Daniel Lezcano <daniel.lezcano@linaro.org>
	*
	* The idle injection framework provides a way to force CPUs to enter idle
	* states for a specified fraction of time over a specified period.
	*
	* It relies on the smpboot kthreads feature providing common code for CPU
	* hotplug and thread [un]parking.
	*
	* All of the kthreads used for idle injection are created at init time.
	*
	* Next, the users of the idle injection framework provide a cpumask via
	* its register function. The kthreads will be synchronized with respect to
	* this cpumask.
	*
	* The idle + run duration is specified via separate helpers and that allows
	* idle injection to be started.
	*
	* The idle injection kthreads will call play_idle_precise() with the idle
	* duration and max allowed latency specified as per the above.
	*
	* After all of them have been woken up, a timer is set to start the next idle
	* injection cycle.
	*
	* The timer interrupt handler will wake up the idle injection kthreads for
	* all of the CPUs in the cpumask provided by the user.
	*
	* Idle injection is stopped synchronously and no leftover idle injection
	* kthread activity after its completion is guaranteed.
	*
	* It is up to the user of this framework to provide a lock for higher-level
	* synchronization to prevent race conditions like starting idle injection
	* while unregistering from the framework.
	*/
	#define pr_fmt(fmt) "ii_dev: " fmt

	#include <linux/cpu.h>
	#include <linux/hrtimer.h>
	#include <linux/kthread.h>
	#include <linux/sched.h>
	#include <linux/slab.h>
	#include <linux/smpboot.h>
	#include <linux/idle_inject.h>

	#include <uapi/linux/sched/types.h>

	/**
	* struct idle_inject_thread - task on/off switch structure
	* @tsk: task injecting the idle cycles
	* @should_run: whether or not to run the task (for the smpboot kthread API)
	*/
	struct idle_inject_thread {
	struct task_struct *tsk;
	int should_run;
	};

	/**
	* struct idle_inject_device - idle injection data
	* @timer: idle injection period timer
	* @idle_duration_us: duration of CPU idle time to inject
	* @run_duration_us: duration of CPU run time to allow
	* @latency_us: max allowed latency
	* @update: Optional callback deciding whether or not to skip idle
	* injection in the given cycle.
	* @cpumask: mask of CPUs affected by idle injection
	*
	* This structure is used to define per instance idle inject device data. Each
	* instance has an idle duration, a run duration and mask of CPUs to inject
	* idle.
	*
	* Actual CPU idle time is injected by calling kernel scheduler interface
	* play_idle_precise(). There is one optional callback that can be registered
	* by calling idle_inject_register_full():
	*
	* update() - This callback is invoked just before waking up CPUs to inject
	* idle. If it returns false, CPUs are not woken up to inject idle in the given
	* cycle. It also allows the caller to readjust the idle and run duration by
	* calling idle_inject_set_duration() for the next cycle.
	*/
	struct idle_inject_device {
	struct hrtimer timer;
	unsigned int idle_duration_us;
	unsigned int run_duration_us;
	unsigned int latency_us;
	bool (*update)(void);
	unsigned long cpumask[];
	};

	static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
	static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);

	/**
	* idle_inject_wakeup - Wake up idle injection threads
	* @ii_dev: target idle injection device
	*
	* Every idle injection task associated with the given idle injection device
	* and running on an online CPU will be woken up.
	*/
	static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
	{
	struct idle_inject_thread *iit;
	unsigned int cpu;

	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
	iit = per_cpu_ptr(&idle_inject_thread, cpu);
	iit->should_run = 1;
	wake_up_process(iit->tsk);
	}
	}

	/**
	* idle_inject_timer_fn - idle injection timer function
	* @timer: idle injection hrtimer
	*
	* This function is called when the idle injection timer expires. It wakes up
	* idle injection tasks associated with the timer and they, in turn, invoke
	* play_idle_precise() to inject a specified amount of CPU idle time.
	*
	* Return: HRTIMER_RESTART.
	*/
	static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
	{
	unsigned int duration_us;
	struct idle_inject_device *ii_dev =
	container_of(timer, struct idle_inject_device, timer);

	if (!ii_dev->update \|\| (ii_dev->update && ii_dev->update()))
	idle_inject_wakeup(ii_dev);

	duration_us = READ_ONCE(ii_dev->run_duration_us);
	duration_us += READ_ONCE(ii_dev->idle_duration_us);

	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));

	return HRTIMER_RESTART;
	}

	/**
	* idle_inject_fn - idle injection work function
	* @cpu: the CPU owning the task
	*
	* This function calls play_idle_precise() to inject a specified amount of CPU
	* idle time.
	*/
	static void idle_inject_fn(unsigned int cpu)
	{
	struct idle_inject_device *ii_dev;
	struct idle_inject_thread *iit;

	ii_dev = per_cpu(idle_inject_device, cpu);
	iit = per_cpu_ptr(&idle_inject_thread, cpu);

	/*
	* Let the smpboot main loop know that the task should not run again.
	*/
	iit->should_run = 0;

	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
	READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
	}

	/**
	* idle_inject_set_duration - idle and run duration update helper
	* @ii_dev: idle injection control device structure
	* @run_duration_us: CPU run time to allow in microseconds
	* @idle_duration_us: CPU idle time to inject in microseconds
	*/
	void idle_inject_set_duration(struct idle_inject_device *ii_dev,
	unsigned int run_duration_us,
	unsigned int idle_duration_us)
	{
	if (run_duration_us + idle_duration_us) {
	WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
	WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
	}
	if (!run_duration_us)
	pr_debug("CPU is forced to 100 percent idle\n");
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);

	/**
	* idle_inject_get_duration - idle and run duration retrieval helper
	* @ii_dev: idle injection control device structure
	* @run_duration_us: memory location to store the current CPU run time
	* @idle_duration_us: memory location to store the current CPU idle time
	*/
	void idle_inject_get_duration(struct idle_inject_device *ii_dev,
	unsigned int *run_duration_us,
	unsigned int *idle_duration_us)
	{
	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);

	/**
	* idle_inject_set_latency - set the maximum latency allowed
	* @ii_dev: idle injection control device structure
	* @latency_us: set the latency requirement for the idle state
	*/
	void idle_inject_set_latency(struct idle_inject_device *ii_dev,
	unsigned int latency_us)
	{
	WRITE_ONCE(ii_dev->latency_us, latency_us);
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);

	/**
	* idle_inject_start - start idle injections
	* @ii_dev: idle injection control device structure
	*
	* The function starts idle injection by first waking up all of the idle
	* injection kthreads associated with @ii_dev to let them inject CPU idle time
	* sets up a timer to start the next idle injection period.
	*
	* Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
	*/
	int idle_inject_start(struct idle_inject_device *ii_dev)
	{
	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);

	if (!(idle_duration_us + run_duration_us))
	return -EINVAL;

	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
	cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

	idle_inject_wakeup(ii_dev);

	hrtimer_start(&ii_dev->timer,
	ns_to_ktime((idle_duration_us + run_duration_us) *
	NSEC_PER_USEC),
	HRTIMER_MODE_REL);

	return 0;
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);

	/**
	* idle_inject_stop - stops idle injections
	* @ii_dev: idle injection control device structure
	*
	* The function stops idle injection and waits for the threads to finish work.
	* If CPU idle time is being injected when this function runs, then it will
	* wait until the end of the cycle.
	*
	* When it returns, there is no more idle injection kthread activity. The
	* kthreads are scheduled out and the periodic timer is off.
	*/
	void idle_inject_stop(struct idle_inject_device *ii_dev)
	{
	struct idle_inject_thread *iit;
	unsigned int cpu;

	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
	cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

	hrtimer_cancel(&ii_dev->timer);

	/*
	* Stopping idle injection requires all of the idle injection kthreads
	* associated with the given cpumask to be parked and stay that way, so
	* prevent CPUs from going online at this point. Any CPUs going online
	* after the loop below will be covered by clearing the should_run flag
	* that will cause the smpboot main loop to schedule them out.
	*/
	cpu_hotplug_disable();

	/*
	* Iterate over all (online + offline) CPUs here in case one of them
	* goes offline with the should_run flag set so as to prevent its idle
	* injection kthread from running when the CPU goes online again after
	* the ii_dev has been freed.
	*/
	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
	iit = per_cpu_ptr(&idle_inject_thread, cpu);
	iit->should_run = 0;

	wait_task_inactive(iit->tsk, TASK_ANY);
	}

	cpu_hotplug_enable();
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);

	/**
	* idle_inject_setup - prepare the current task for idle injection
	* @cpu: not used
	*
	* Called once, this function is in charge of setting the current task's
	* scheduler parameters to make it an RT task.
	*/
	static void idle_inject_setup(unsigned int cpu)
	{
	sched_set_fifo(current);
	}

	/**
	* idle_inject_should_run - function helper for the smpboot API
	* @cpu: CPU the kthread is running on
	*
	* Return: whether or not the thread can run.
	*/
	static int idle_inject_should_run(unsigned int cpu)
	{
	struct idle_inject_thread *iit =
	per_cpu_ptr(&idle_inject_thread, cpu);

	return iit->should_run;
	}

	/**
	* idle_inject_register_full - initialize idle injection on a set of CPUs
	* @cpumask: CPUs to be affected by idle injection
	* @update: This callback is called just before waking up CPUs to inject
	* idle
	*
	* This function creates an idle injection control device structure for the
	* given set of CPUs and initializes the timer associated with it. This
	* function also allows to register update()callback.
	* It does not start any injection cycles.
	*
	* Return: NULL if memory allocation fails, idle injection control device
	* pointer on success.
	*/

	struct idle_inject_device idle_inject_register_full(struct cpumask cpumask,
	bool (*update)(void))
	{
	struct idle_inject_device *ii_dev;
	int cpu, cpu_rb;

	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
	if (!ii_dev)
	return NULL;

	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	ii_dev->timer.function = idle_inject_timer_fn;
	ii_dev->latency_us = UINT_MAX;
	ii_dev->update = update;

	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {

	if (per_cpu(idle_inject_device, cpu)) {
	pr_err("cpu%d is already registered\n", cpu);
	goto out_rollback;
	}

	per_cpu(idle_inject_device, cpu) = ii_dev;
	}

	return ii_dev;

	out_rollback:
	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
	if (cpu == cpu_rb)
	break;
	per_cpu(idle_inject_device, cpu_rb) = NULL;
	}

	kfree(ii_dev);

	return NULL;
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);

	/**
	* idle_inject_register - initialize idle injection on a set of CPUs
	* @cpumask: CPUs to be affected by idle injection
	*
	* This function creates an idle injection control device structure for the
	* given set of CPUs and initializes the timer associated with it. It does not
	* start any injection cycles.
	*
	* Return: NULL if memory allocation fails, idle injection control device
	* pointer on success.
	*/
	struct idle_inject_device idle_inject_register(struct cpumask cpumask)
	{
	return idle_inject_register_full(cpumask, NULL);
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);

	/**
	* idle_inject_unregister - unregister idle injection control device
	* @ii_dev: idle injection control device to unregister
	*
	* The function stops idle injection for the given control device,
	* unregisters its kthreads and frees memory allocated when that device was
	* created.
	*/
	void idle_inject_unregister(struct idle_inject_device *ii_dev)
	{
	unsigned int cpu;

	idle_inject_stop(ii_dev);

	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
	per_cpu(idle_inject_device, cpu) = NULL;

	kfree(ii_dev);
	}
	EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);

	static struct smp_hotplug_thread idle_inject_threads = {
	.store = &idle_inject_thread.tsk,
	.setup = idle_inject_setup,
	.thread_fn = idle_inject_fn,
	.thread_comm = "idle_inject/%u",
	.thread_should_run = idle_inject_should_run,
	};

	static int __init idle_inject_init(void)
	{
	return smpboot_register_percpu_thread(&idle_inject_threads);
	}
	early_initcall(idle_inject_init);