Thomas Gleixner | 7925f8f | 2019-05-28 10:10:18 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 2 | /* |
| 3 | * menu.c - the menu idle governor |
| 4 | * |
| 5 | * Copyright (C) 2006-2007 Adam Belay <abelay@novell.com> |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 6 | * Copyright (C) 2009 Intel Corporation |
| 7 | * Author: |
| 8 | * Arjan van de Ven <arjan@linux.intel.com> |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 9 | */ |
| 10 | |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/cpuidle.h> |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 13 | #include <linux/time.h> |
| 14 | #include <linux/ktime.h> |
| 15 | #include <linux/hrtimer.h> |
| 16 | #include <linux/tick.h> |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 17 | #include <linux/sched.h> |
Ingo Molnar | 4f17722 | 2017-02-08 08:45:17 +0100 | [diff] [blame] | 18 | #include <linux/sched/loadavg.h> |
Ingo Molnar | 03441a3 | 2017-02-08 18:51:35 +0100 | [diff] [blame] | 19 | #include <linux/sched/stat.h> |
Stephen Hemminger | 5787536 | 2010-01-08 14:43:08 -0800 | [diff] [blame] | 20 | #include <linux/math64.h> |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 21 | |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 22 | #define BUCKETS 12 |
Mel Gorman | ae77930 | 2014-08-06 14:19:18 +0100 | [diff] [blame] | 23 | #define INTERVAL_SHIFT 3 |
| 24 | #define INTERVALS (1UL << INTERVAL_SHIFT) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 25 | #define RESOLUTION 1024 |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 26 | #define DECAY 8 |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 27 | #define MAX_INTERESTING (50000 * NSEC_PER_USEC) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 28 | |
| 29 | /* |
| 30 | * Concepts and ideas behind the menu governor |
| 31 | * |
| 32 | * For the menu governor, there are 3 decision factors for picking a C |
| 33 | * state: |
| 34 | * 1) Energy break even point |
| 35 | * 2) Performance impact |
| 36 | * 3) Latency tolerance (from pmqos infrastructure) |
| 37 | * These these three factors are treated independently. |
| 38 | * |
| 39 | * Energy break even point |
| 40 | * ----------------------- |
| 41 | * C state entry and exit have an energy cost, and a certain amount of time in |
| 42 | * the C state is required to actually break even on this cost. CPUIDLE |
| 43 | * provides us this duration in the "target_residency" field. So all that we |
| 44 | * need is a good prediction of how long we'll be idle. Like the traditional |
| 45 | * menu governor, we start with the actual known "next timer event" time. |
| 46 | * |
| 47 | * Since there are other source of wakeups (interrupts for example) than |
| 48 | * the next timer event, this estimation is rather optimistic. To get a |
| 49 | * more realistic estimate, a correction factor is applied to the estimate, |
| 50 | * that is based on historic behavior. For example, if in the past the actual |
| 51 | * duration always was 50% of the next timer tick, the correction factor will |
| 52 | * be 0.5. |
| 53 | * |
| 54 | * menu uses a running average for this correction factor, however it uses a |
| 55 | * set of factors, not just a single factor. This stems from the realization |
| 56 | * that the ratio is dependent on the order of magnitude of the expected |
| 57 | * duration; if we expect 500 milliseconds of idle time the likelihood of |
| 58 | * getting an interrupt very early is much higher than if we expect 50 micro |
| 59 | * seconds of idle time. A second independent factor that has big impact on |
| 60 | * the actual factor is if there is (disk) IO outstanding or not. |
| 61 | * (as a special twist, we consider every sleep longer than 50 milliseconds |
| 62 | * as perfect; there are no power gains for sleeping longer than this) |
| 63 | * |
| 64 | * For these two reasons we keep an array of 12 independent factors, that gets |
| 65 | * indexed based on the magnitude of the expected duration as well as the |
| 66 | * "is IO outstanding" property. |
| 67 | * |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 68 | * Repeatable-interval-detector |
| 69 | * ---------------------------- |
| 70 | * There are some cases where "next timer" is a completely unusable predictor: |
| 71 | * Those cases where the interval is fixed, for example due to hardware |
| 72 | * interrupt mitigation, but also due to fixed transfer rate devices such as |
| 73 | * mice. |
| 74 | * For this, we use a different predictor: We track the duration of the last 8 |
| 75 | * intervals and if the stand deviation of these 8 intervals is below a |
| 76 | * threshold value, we use the average of these intervals as prediction. |
| 77 | * |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 78 | * Limiting Performance Impact |
| 79 | * --------------------------- |
| 80 | * C states, especially those with large exit latencies, can have a real |
Lucas De Marchi | 20e3341 | 2010-09-07 12:53:49 -0400 | [diff] [blame] | 81 | * noticeable impact on workloads, which is not acceptable for most sysadmins, |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 82 | * and in addition, less performance has a power price of its own. |
| 83 | * |
| 84 | * As a general rule of thumb, menu assumes that the following heuristic |
| 85 | * holds: |
| 86 | * The busier the system, the less impact of C states is acceptable |
| 87 | * |
| 88 | * This rule-of-thumb is implemented using a performance-multiplier: |
| 89 | * If the exit latency times the performance multiplier is longer than |
| 90 | * the predicted duration, the C state is not considered a candidate |
| 91 | * for selection due to a too high performance impact. So the higher |
| 92 | * this multiplier is, the longer we need to be idle to pick a deep C |
| 93 | * state, and thus the less likely a busy CPU will hit such a deep |
| 94 | * C state. |
| 95 | * |
| 96 | * Two factors are used in determing this multiplier: |
| 97 | * a value of 10 is added for each point of "per cpu load average" we have. |
| 98 | * a value of 5 points is added for each process that is waiting for |
| 99 | * IO on this CPU. |
| 100 | * (these values are experimentally determined) |
| 101 | * |
| 102 | * The load average factor gives a longer term (few seconds) input to the |
| 103 | * decision, while the iowait value gives a cpu local instantanious input. |
| 104 | * The iowait factor may look low, but realize that this is also already |
| 105 | * represented in the system load average. |
| 106 | * |
| 107 | */ |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 108 | |
| 109 | struct menu_device { |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 110 | int needs_update; |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 111 | int tick_wakeup; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 112 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 113 | u64 next_timer_ns; |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 114 | unsigned int bucket; |
Tuukka Tikkanen | 51f245b | 2013-08-14 19:02:41 +0300 | [diff] [blame] | 115 | unsigned int correction_factor[BUCKETS]; |
Tuukka Tikkanen | 939e33b | 2013-08-14 19:02:38 +0300 | [diff] [blame] | 116 | unsigned int intervals[INTERVALS]; |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 117 | int interval_ptr; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 118 | }; |
| 119 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 120 | static inline int which_bucket(u64 duration_ns, unsigned long nr_iowaiters) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 121 | { |
| 122 | int bucket = 0; |
| 123 | |
| 124 | /* |
| 125 | * We keep two groups of stats; one with no |
| 126 | * IO pending, one without. |
| 127 | * This allows us to calculate |
| 128 | * E(duration)|iowait |
| 129 | */ |
Mel Gorman | 64b4ca5 | 2014-08-06 14:19:20 +0100 | [diff] [blame] | 130 | if (nr_iowaiters) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 131 | bucket = BUCKETS/2; |
| 132 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 133 | if (duration_ns < 10ULL * NSEC_PER_USEC) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 134 | return bucket; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 135 | if (duration_ns < 100ULL * NSEC_PER_USEC) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 136 | return bucket + 1; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 137 | if (duration_ns < 1000ULL * NSEC_PER_USEC) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 138 | return bucket + 2; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 139 | if (duration_ns < 10000ULL * NSEC_PER_USEC) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 140 | return bucket + 3; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 141 | if (duration_ns < 100000ULL * NSEC_PER_USEC) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 142 | return bucket + 4; |
| 143 | return bucket + 5; |
| 144 | } |
| 145 | |
| 146 | /* |
| 147 | * Return a multiplier for the exit latency that is intended |
| 148 | * to take performance requirements into account. |
| 149 | * The more performance critical we estimate the system |
| 150 | * to be, the higher this multiplier, and thus the higher |
| 151 | * the barrier to go to an expensive C state. |
| 152 | */ |
Daniel Lezcano | a7fe519 | 2018-10-04 14:04:03 +0200 | [diff] [blame] | 153 | static inline int performance_multiplier(unsigned long nr_iowaiters) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 154 | { |
Daniel Lezcano | a7fe519 | 2018-10-04 14:04:03 +0200 | [diff] [blame] | 155 | /* for IO wait tasks (per cpu!) we add 10x each */ |
| 156 | return 1 + 10 * nr_iowaiters; |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 157 | } |
| 158 | |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 159 | static DEFINE_PER_CPU(struct menu_device, menu_devices); |
| 160 | |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 161 | static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev); |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 162 | |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 163 | /* |
| 164 | * Try detecting repeating patterns by keeping track of the last 8 |
| 165 | * intervals, and checking if the standard deviation of that set |
| 166 | * of points is below a threshold. If it is... then use the |
| 167 | * average of these 8 points as the estimated value. |
| 168 | */ |
Rafael J. Wysocki | f1c8e41 | 2018-10-15 13:53:25 +0200 | [diff] [blame] | 169 | static unsigned int get_typical_interval(struct menu_device *data, |
| 170 | unsigned int predicted_us) |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 171 | { |
Tuukka Tikkanen | 4cd46bc | 2013-08-14 19:02:37 +0300 | [diff] [blame] | 172 | int i, divisor; |
Rafael J. Wysocki | f1c8e41 | 2018-10-15 13:53:25 +0200 | [diff] [blame] | 173 | unsigned int min, max, thresh, avg; |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 174 | uint64_t sum, variance; |
Tuukka Tikkanen | 0e96d5a | 2013-08-14 19:02:39 +0300 | [diff] [blame] | 175 | |
Rafael J. Wysocki | 814b8797 | 2019-02-27 14:35:50 +0100 | [diff] [blame] | 176 | thresh = INT_MAX; /* Discard outliers above this value */ |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 177 | |
| 178 | again: |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 179 | |
Tuukka Tikkanen | 0e96d5a | 2013-08-14 19:02:39 +0300 | [diff] [blame] | 180 | /* First calculate the average of past intervals */ |
Rafael J. Wysocki | f1c8e41 | 2018-10-15 13:53:25 +0200 | [diff] [blame] | 181 | min = UINT_MAX; |
Tuukka Tikkanen | 4cd46bc | 2013-08-14 19:02:37 +0300 | [diff] [blame] | 182 | max = 0; |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 183 | sum = 0; |
Tuukka Tikkanen | 4cd46bc | 2013-08-14 19:02:37 +0300 | [diff] [blame] | 184 | divisor = 0; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 185 | for (i = 0; i < INTERVALS; i++) { |
Tuukka Tikkanen | 0e96d5a | 2013-08-14 19:02:39 +0300 | [diff] [blame] | 186 | unsigned int value = data->intervals[i]; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 187 | if (value <= thresh) { |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 188 | sum += value; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 189 | divisor++; |
| 190 | if (value > max) |
| 191 | max = value; |
Rafael J. Wysocki | f1c8e41 | 2018-10-15 13:53:25 +0200 | [diff] [blame] | 192 | |
| 193 | if (value < min) |
| 194 | min = value; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 195 | } |
| 196 | } |
Rafael J. Wysocki | f1c8e41 | 2018-10-15 13:53:25 +0200 | [diff] [blame] | 197 | |
| 198 | /* |
| 199 | * If the result of the computation is going to be discarded anyway, |
| 200 | * avoid the computation altogether. |
| 201 | */ |
| 202 | if (min >= predicted_us) |
| 203 | return UINT_MAX; |
| 204 | |
Mel Gorman | ae77930 | 2014-08-06 14:19:18 +0100 | [diff] [blame] | 205 | if (divisor == INTERVALS) |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 206 | avg = sum >> INTERVAL_SHIFT; |
Mel Gorman | ae77930 | 2014-08-06 14:19:18 +0100 | [diff] [blame] | 207 | else |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 208 | avg = div_u64(sum, divisor); |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 209 | |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 210 | /* Then try to determine variance */ |
| 211 | variance = 0; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 212 | for (i = 0; i < INTERVALS; i++) { |
Tuukka Tikkanen | 0e96d5a | 2013-08-14 19:02:39 +0300 | [diff] [blame] | 213 | unsigned int value = data->intervals[i]; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 214 | if (value <= thresh) { |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 215 | int64_t diff = (int64_t)value - avg; |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 216 | variance += diff * diff; |
Youquan Song | c96ca4f | 2012-10-26 12:27:07 +0200 | [diff] [blame] | 217 | } |
| 218 | } |
Mel Gorman | ae77930 | 2014-08-06 14:19:18 +0100 | [diff] [blame] | 219 | if (divisor == INTERVALS) |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 220 | variance >>= INTERVAL_SHIFT; |
Mel Gorman | ae77930 | 2014-08-06 14:19:18 +0100 | [diff] [blame] | 221 | else |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 222 | do_div(variance, divisor); |
Mel Gorman | ae77930 | 2014-08-06 14:19:18 +0100 | [diff] [blame] | 223 | |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 224 | /* |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 225 | * The typical interval is obtained when standard deviation is |
| 226 | * small (stddev <= 20 us, variance <= 400 us^2) or standard |
| 227 | * deviation is small compared to the average interval (avg > |
| 228 | * 6*stddev, avg^2 > 36*variance). The average is smaller than |
| 229 | * UINT_MAX aka U32_MAX, so computing its square does not |
| 230 | * overflow a u64. We simply reject this candidate average if |
| 231 | * the standard deviation is greater than 715 s (which is |
| 232 | * rather unlikely). |
Tuukka Tikkanen | 0d6a7ff | 2013-08-14 19:02:36 +0300 | [diff] [blame] | 233 | * |
Tuukka Tikkanen | 330647a | 2013-08-14 19:02:34 +0300 | [diff] [blame] | 234 | * Use this result only if there is no timer to wake us up sooner. |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 235 | */ |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 236 | if (likely(variance <= U64_MAX/36)) { |
Rasmus Villemoes | 3b99669 | 2016-02-16 20:19:19 +0100 | [diff] [blame] | 237 | if ((((u64)avg*avg > variance*36) && (divisor * 4 >= INTERVALS * 3)) |
Rasmus Villemoes | 7024b18 | 2016-02-16 20:19:18 +0100 | [diff] [blame] | 238 | || variance <= 400) { |
Rik van Riel | e132b9b | 2016-03-16 12:14:00 -0400 | [diff] [blame] | 239 | return avg; |
Tuukka Tikkanen | 0d6a7ff | 2013-08-14 19:02:36 +0300 | [diff] [blame] | 240 | } |
Youquan Song | 69a37be | 2012-10-26 12:26:41 +0200 | [diff] [blame] | 241 | } |
Tuukka Tikkanen | 017099e | 2013-08-14 19:02:35 +0300 | [diff] [blame] | 242 | |
| 243 | /* |
| 244 | * If we have outliers to the upside in our distribution, discard |
| 245 | * those by setting the threshold to exclude these outliers, then |
| 246 | * calculate the average and standard deviation again. Once we get |
| 247 | * down to the bottom 3/4 of our samples, stop excluding samples. |
| 248 | * |
| 249 | * This can deal with workloads that have long pauses interspersed |
| 250 | * with sporadic activity with a bunch of short pauses. |
| 251 | */ |
| 252 | if ((divisor * 4) <= INTERVALS * 3) |
Rik van Riel | e132b9b | 2016-03-16 12:14:00 -0400 | [diff] [blame] | 253 | return UINT_MAX; |
Tuukka Tikkanen | 017099e | 2013-08-14 19:02:35 +0300 | [diff] [blame] | 254 | |
| 255 | thresh = max - 1; |
| 256 | goto again; |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 257 | } |
| 258 | |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 259 | /** |
| 260 | * menu_select - selects the next idle state to enter |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 261 | * @drv: cpuidle driver containing state data |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 262 | * @dev: the CPU |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 263 | * @stop_tick: indication on whether or not to stop the tick |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 264 | */ |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 265 | static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, |
| 266 | bool *stop_tick) |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 267 | { |
Christoph Lameter | 229b686 | 2014-08-17 12:30:30 -0500 | [diff] [blame] | 268 | struct menu_device *data = this_cpu_ptr(&menu_devices); |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 269 | s64 latency_req = cpuidle_governor_latency_req(dev->cpu); |
Rafael J. Wysocki | 03dba27 | 2018-10-01 11:56:21 +0200 | [diff] [blame] | 270 | unsigned int predicted_us; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 271 | u64 predicted_ns; |
| 272 | u64 interactivity_req; |
Daniel Lezcano | a7fe519 | 2018-10-04 14:04:03 +0200 | [diff] [blame] | 273 | unsigned long nr_iowaiters; |
Rafael J. Wysocki | 296bb1e | 2018-04-05 19:12:34 +0200 | [diff] [blame] | 274 | ktime_t delta_next; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 275 | int i, idx; |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 276 | |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 277 | if (data->needs_update) { |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 278 | menu_update(drv, dev); |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 279 | data->needs_update = 0; |
| 280 | } |
| 281 | |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 282 | /* determine the expected residency time, round up */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 283 | data->next_timer_ns = tick_nohz_get_sleep_length(&delta_next); |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 284 | |
Daniel Lezcano | a7fe519 | 2018-10-04 14:04:03 +0200 | [diff] [blame] | 285 | nr_iowaiters = nr_iowait_cpu(dev->cpu); |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 286 | data->bucket = which_bucket(data->next_timer_ns, nr_iowaiters); |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 287 | |
Rafael J. Wysocki | 53812cd | 2018-10-02 23:47:43 +0200 | [diff] [blame] | 288 | if (unlikely(drv->state_count <= 1 || latency_req == 0) || |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 289 | ((data->next_timer_ns < drv->states[1].target_residency_ns || |
| 290 | latency_req < drv->states[1].exit_latency_ns) && |
Rafael J. Wysocki | 99e98d3 | 2019-11-04 12:16:17 +0100 | [diff] [blame] | 291 | !dev->states_usage[0].disable)) { |
Rafael J. Wysocki | 8b007eb | 2018-10-02 23:46:28 +0200 | [diff] [blame] | 292 | /* |
| 293 | * In this case state[0] will be used no matter what, so return |
Rafael J. Wysocki | 32b91ca | 2019-07-18 10:53:21 +0200 | [diff] [blame] | 294 | * it right away and keep the tick running if state[0] is a |
| 295 | * polling one. |
Rafael J. Wysocki | 8b007eb | 2018-10-02 23:46:28 +0200 | [diff] [blame] | 296 | */ |
Rafael J. Wysocki | 32b91ca | 2019-07-18 10:53:21 +0200 | [diff] [blame] | 297 | *stop_tick = !(drv->states[0].flags & CPUIDLE_FLAG_POLLING); |
Rafael J. Wysocki | 8b007eb | 2018-10-02 23:46:28 +0200 | [diff] [blame] | 298 | return 0; |
| 299 | } |
| 300 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 301 | /* Round up the result for half microseconds. */ |
| 302 | predicted_us = div_u64(data->next_timer_ns * |
| 303 | data->correction_factor[data->bucket] + |
| 304 | (RESOLUTION * DECAY * NSEC_PER_USEC) / 2, |
| 305 | RESOLUTION * DECAY * NSEC_PER_USEC); |
| 306 | /* Use the lowest expected idle interval to pick the idle state. */ |
| 307 | predicted_ns = (u64)min(predicted_us, |
| 308 | get_typical_interval(data, predicted_us)) * |
| 309 | NSEC_PER_USEC; |
Rik van Riel | e132b9b | 2016-03-16 12:14:00 -0400 | [diff] [blame] | 310 | |
Rafael J. Wysocki | 87c9fe6 | 2018-04-05 19:12:43 +0200 | [diff] [blame] | 311 | if (tick_nohz_tick_stopped()) { |
| 312 | /* |
| 313 | * If the tick is already stopped, the cost of possible short |
| 314 | * idle duration misprediction is much higher, because the CPU |
| 315 | * may be stuck in a shallow idle state for a long time as a |
Rafael J. Wysocki | 5ef499c | 2018-08-14 12:34:40 +0200 | [diff] [blame] | 316 | * result of it. In that case say we might mispredict and use |
| 317 | * the known time till the closest timer event for the idle |
| 318 | * state selection. |
Rafael J. Wysocki | 87c9fe6 | 2018-04-05 19:12:43 +0200 | [diff] [blame] | 319 | */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 320 | if (predicted_ns < TICK_NSEC) |
| 321 | predicted_ns = delta_next; |
Rafael J. Wysocki | 87c9fe6 | 2018-04-05 19:12:43 +0200 | [diff] [blame] | 322 | } else { |
| 323 | /* |
| 324 | * Use the performance multiplier and the user-configurable |
| 325 | * latency_req to determine the maximum exit latency. |
| 326 | */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 327 | interactivity_req = div64_u64(predicted_ns, |
| 328 | performance_multiplier(nr_iowaiters)); |
Rafael J. Wysocki | 87c9fe6 | 2018-04-05 19:12:43 +0200 | [diff] [blame] | 329 | if (latency_req > interactivity_req) |
| 330 | latency_req = interactivity_req; |
| 331 | } |
Rik van Riel | e132b9b | 2016-03-16 12:14:00 -0400 | [diff] [blame] | 332 | |
| 333 | /* |
Ai Li | 71abbbf | 2010-08-09 17:20:13 -0700 | [diff] [blame] | 334 | * Find the idle state with the lowest power while satisfying |
| 335 | * our constraints. |
| 336 | */ |
Nicholas Piggin | 3ed09c9 | 2017-06-26 15:38:15 +1000 | [diff] [blame] | 337 | idx = -1; |
Rafael J. Wysocki | 96c3d11 | 2018-10-02 23:44:06 +0200 | [diff] [blame] | 338 | for (i = 0; i < drv->state_count; i++) { |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 339 | struct cpuidle_state *s = &drv->states[i]; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 340 | |
Rafael J. Wysocki | 99e98d3 | 2019-11-04 12:16:17 +0100 | [diff] [blame] | 341 | if (dev->states_usage[i].disable) |
ShuoX Liu | 3a53396b | 2012-03-28 15:19:11 -0700 | [diff] [blame] | 342 | continue; |
Rafael J. Wysocki | 96c3d11 | 2018-10-02 23:44:06 +0200 | [diff] [blame] | 343 | |
Nicholas Piggin | 3ed09c9 | 2017-06-26 15:38:15 +1000 | [diff] [blame] | 344 | if (idx == -1) |
| 345 | idx = i; /* first enabled state */ |
Rafael J. Wysocki | 96c3d11 | 2018-10-02 23:44:06 +0200 | [diff] [blame] | 346 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 347 | if (s->target_residency_ns > predicted_ns) { |
Rafael J. Wysocki | 96c3d11 | 2018-10-02 23:44:06 +0200 | [diff] [blame] | 348 | /* |
| 349 | * Use a physical idle state, not busy polling, unless |
Rafael J. Wysocki | bde091e | 2018-10-10 14:16:38 +0200 | [diff] [blame] | 350 | * a timer is going to trigger soon enough. |
Rafael J. Wysocki | 96c3d11 | 2018-10-02 23:44:06 +0200 | [diff] [blame] | 351 | */ |
| 352 | if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 353 | s->exit_latency_ns <= latency_req && |
| 354 | s->target_residency_ns <= data->next_timer_ns) { |
| 355 | predicted_ns = s->target_residency_ns; |
Rafael J. Wysocki | 96c3d11 | 2018-10-02 23:44:06 +0200 | [diff] [blame] | 356 | idx = i; |
| 357 | break; |
| 358 | } |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 359 | if (predicted_ns < TICK_NSEC) |
Rafael J. Wysocki | 5ef499c | 2018-08-14 12:34:40 +0200 | [diff] [blame] | 360 | break; |
| 361 | |
Rafael J. Wysocki | 757ab15 | 2018-08-21 10:44:10 +0200 | [diff] [blame] | 362 | if (!tick_nohz_tick_stopped()) { |
| 363 | /* |
| 364 | * If the state selected so far is shallow, |
| 365 | * waking up early won't hurt, so retain the |
| 366 | * tick in that case and let the governor run |
| 367 | * again in the next iteration of the loop. |
| 368 | */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 369 | predicted_ns = drv->states[idx].target_residency_ns; |
Rafael J. Wysocki | 757ab15 | 2018-08-21 10:44:10 +0200 | [diff] [blame] | 370 | break; |
| 371 | } |
| 372 | |
Rafael J. Wysocki | 5ef499c | 2018-08-14 12:34:40 +0200 | [diff] [blame] | 373 | /* |
| 374 | * If the state selected so far is shallow and this |
| 375 | * state's target residency matches the time till the |
| 376 | * closest timer event, select this one to avoid getting |
| 377 | * stuck in the shallow one for too long. |
| 378 | */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 379 | if (drv->states[idx].target_residency_ns < TICK_NSEC && |
| 380 | s->target_residency_ns <= delta_next) |
Rafael J. Wysocki | 5ef499c | 2018-08-14 12:34:40 +0200 | [diff] [blame] | 381 | idx = i; |
| 382 | |
Rafael J. Wysocki | eb40a38 | 2018-10-02 23:45:07 +0200 | [diff] [blame] | 383 | return idx; |
Rafael J. Wysocki | 5ef499c | 2018-08-14 12:34:40 +0200 | [diff] [blame] | 384 | } |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 385 | if (s->exit_latency_ns > latency_req) |
Alex Shi | 8e37e1a | 2017-01-12 21:27:02 +0800 | [diff] [blame] | 386 | break; |
Rafael J. Wysocki | 32b91ca | 2019-07-18 10:53:21 +0200 | [diff] [blame] | 387 | |
Nicholas Piggin | 3ed09c9 | 2017-06-26 15:38:15 +1000 | [diff] [blame] | 388 | idx = i; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 389 | } |
| 390 | |
Nicholas Piggin | 3ed09c9 | 2017-06-26 15:38:15 +1000 | [diff] [blame] | 391 | if (idx == -1) |
| 392 | idx = 0; /* No states enabled. Must use 0. */ |
| 393 | |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 394 | /* |
| 395 | * Don't stop the tick if the selected state is a polling one or if the |
| 396 | * expected idle duration is shorter than the tick period length. |
| 397 | */ |
Rafael J. Wysocki | 5ef499c | 2018-08-14 12:34:40 +0200 | [diff] [blame] | 398 | if (((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) || |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 399 | predicted_ns < TICK_NSEC) && !tick_nohz_tick_stopped()) { |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 400 | *stop_tick = false; |
| 401 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 402 | if (idx > 0 && drv->states[idx].target_residency_ns > delta_next) { |
Rafael J. Wysocki | 296bb1e | 2018-04-05 19:12:34 +0200 | [diff] [blame] | 403 | /* |
| 404 | * The tick is not going to be stopped and the target |
| 405 | * residency of the state to be returned is not within |
| 406 | * the time until the next timer event including the |
| 407 | * tick, so try to correct that. |
| 408 | */ |
| 409 | for (i = idx - 1; i >= 0; i--) { |
Rafael J. Wysocki | 99e98d3 | 2019-11-04 12:16:17 +0100 | [diff] [blame] | 410 | if (dev->states_usage[i].disable) |
Rafael J. Wysocki | 296bb1e | 2018-04-05 19:12:34 +0200 | [diff] [blame] | 411 | continue; |
| 412 | |
| 413 | idx = i; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 414 | if (drv->states[i].target_residency_ns <= delta_next) |
Rafael J. Wysocki | 296bb1e | 2018-04-05 19:12:34 +0200 | [diff] [blame] | 415 | break; |
| 416 | } |
| 417 | } |
| 418 | } |
| 419 | |
Rafael J. Wysocki | eb40a38 | 2018-10-02 23:45:07 +0200 | [diff] [blame] | 420 | return idx; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 421 | } |
| 422 | |
| 423 | /** |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 424 | * menu_reflect - records that data structures need update |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 425 | * @dev: the CPU |
Deepthi Dharwar | e978aa7 | 2011-10-28 16:20:09 +0530 | [diff] [blame] | 426 | * @index: the index of actual entered state |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 427 | * |
| 428 | * NOTE: it's important to be fast here because this operation will add to |
| 429 | * the overall exit latency. |
| 430 | */ |
Deepthi Dharwar | e978aa7 | 2011-10-28 16:20:09 +0530 | [diff] [blame] | 431 | static void menu_reflect(struct cpuidle_device *dev, int index) |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 432 | { |
Christoph Lameter | 229b686 | 2014-08-17 12:30:30 -0500 | [diff] [blame] | 433 | struct menu_device *data = this_cpu_ptr(&menu_devices); |
Rafael J. Wysocki | a802ea9 | 2015-05-04 22:53:28 +0200 | [diff] [blame] | 434 | |
Marcelo Tosatti | 7d4daee | 2019-07-03 20:51:27 -0300 | [diff] [blame] | 435 | dev->last_state_idx = index; |
Rafael J. Wysocki | a802ea9 | 2015-05-04 22:53:28 +0200 | [diff] [blame] | 436 | data->needs_update = 1; |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 437 | data->tick_wakeup = tick_nohz_idle_got_tick(); |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 438 | } |
| 439 | |
| 440 | /** |
| 441 | * menu_update - attempts to guess what happened after entry |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 442 | * @drv: cpuidle driver containing state data |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 443 | * @dev: the CPU |
| 444 | */ |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 445 | static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) |
Corrado Zoccolo | 672917d | 2009-09-21 17:04:09 -0700 | [diff] [blame] | 446 | { |
Christoph Lameter | 229b686 | 2014-08-17 12:30:30 -0500 | [diff] [blame] | 447 | struct menu_device *data = this_cpu_ptr(&menu_devices); |
Marcelo Tosatti | 7d4daee | 2019-07-03 20:51:27 -0300 | [diff] [blame] | 448 | int last_idx = dev->last_state_idx; |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 449 | struct cpuidle_state *target = &drv->states[last_idx]; |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 450 | u64 measured_ns; |
Tuukka Tikkanen | 51f245b | 2013-08-14 19:02:41 +0300 | [diff] [blame] | 451 | unsigned int new_factor; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 452 | |
| 453 | /* |
tuukka.tikkanen@linaro.org | 61c66d6 | 2014-02-24 08:29:34 +0200 | [diff] [blame] | 454 | * Try to figure out how much time passed between entry to low |
| 455 | * power state and occurrence of the wakeup event. |
| 456 | * |
| 457 | * If the entered idle state didn't support residency measurements, |
Len Brown | 4108b3d | 2014-12-16 01:52:06 -0500 | [diff] [blame] | 458 | * we use them anyway if they are short, and if long, |
| 459 | * truncate to the whole expected time. |
tuukka.tikkanen@linaro.org | 61c66d6 | 2014-02-24 08:29:34 +0200 | [diff] [blame] | 460 | * |
| 461 | * Any measured amount of time will include the exit latency. |
| 462 | * Since we are interested in when the wakeup begun, not when it |
Antonio Ospite | 2fba537 | 2014-06-04 14:03:45 +0200 | [diff] [blame] | 463 | * was completed, we must subtract the exit latency. However, if |
tuukka.tikkanen@linaro.org | 61c66d6 | 2014-02-24 08:29:34 +0200 | [diff] [blame] | 464 | * the measured amount of time is less than the exit latency, |
| 465 | * assume the state was never reached and the exit latency is 0. |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 466 | */ |
Len Brown | 4108b3d | 2014-12-16 01:52:06 -0500 | [diff] [blame] | 467 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 468 | if (data->tick_wakeup && data->next_timer_ns > TICK_NSEC) { |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 469 | /* |
| 470 | * The nohz code said that there wouldn't be any events within |
| 471 | * the tick boundary (if the tick was stopped), but the idle |
| 472 | * duration predictor had a differing opinion. Since the CPU |
| 473 | * was woken up by a tick (that wasn't stopped after all), the |
| 474 | * predictor was not quite right, so assume that the CPU could |
| 475 | * have been idle long (but not forever) to help the idle |
| 476 | * duration predictor do a better job next time. |
| 477 | */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 478 | measured_ns = 9 * MAX_INTERESTING / 10; |
Rafael J. Wysocki | 5f26bdc | 2018-10-02 23:42:02 +0200 | [diff] [blame] | 479 | } else if ((drv->states[last_idx].flags & CPUIDLE_FLAG_POLLING) && |
| 480 | dev->poll_time_limit) { |
| 481 | /* |
| 482 | * The CPU exited the "polling" state due to a time limit, so |
| 483 | * the idle duration prediction leading to the selection of that |
| 484 | * state was inaccurate. If a better prediction had been made, |
| 485 | * the CPU might have been woken up from idle by the next timer. |
| 486 | * Assume that to be the case. |
| 487 | */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 488 | measured_ns = data->next_timer_ns; |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 489 | } else { |
| 490 | /* measured value */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 491 | measured_ns = dev->last_residency_ns; |
Len Brown | 4108b3d | 2014-12-16 01:52:06 -0500 | [diff] [blame] | 492 | |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 493 | /* Deduct exit latency */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 494 | if (measured_ns > 2 * target->exit_latency_ns) |
| 495 | measured_ns -= target->exit_latency_ns; |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 496 | else |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 497 | measured_ns /= 2; |
Rafael J. Wysocki | 45f1ff5 | 2018-03-22 17:50:49 +0100 | [diff] [blame] | 498 | } |
Len Brown | 4108b3d | 2014-12-16 01:52:06 -0500 | [diff] [blame] | 499 | |
| 500 | /* Make sure our coefficients do not exceed unity */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 501 | if (measured_ns > data->next_timer_ns) |
| 502 | measured_ns = data->next_timer_ns; |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 503 | |
Tuukka Tikkanen | 51f245b | 2013-08-14 19:02:41 +0300 | [diff] [blame] | 504 | /* Update our correction ratio */ |
| 505 | new_factor = data->correction_factor[data->bucket]; |
| 506 | new_factor -= new_factor / DECAY; |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 507 | |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 508 | if (data->next_timer_ns > 0 && measured_ns < MAX_INTERESTING) |
| 509 | new_factor += div64_u64(RESOLUTION * measured_ns, |
| 510 | data->next_timer_ns); |
venkatesh.pallipadi@intel.com | 320eee7 | 2008-07-30 19:21:43 -0700 | [diff] [blame] | 511 | else |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 512 | /* |
| 513 | * we were idle so long that we count it as a perfect |
| 514 | * prediction |
| 515 | */ |
| 516 | new_factor += RESOLUTION; |
venkatesh.pallipadi@intel.com | 320eee7 | 2008-07-30 19:21:43 -0700 | [diff] [blame] | 517 | |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 518 | /* |
| 519 | * We don't want 0 as factor; we always want at least |
Tuukka Tikkanen | 51f245b | 2013-08-14 19:02:41 +0300 | [diff] [blame] | 520 | * a tiny bit of estimated time. Fortunately, due to rounding, |
| 521 | * new_factor will stay nonzero regardless of measured_us values |
| 522 | * and the compiler can eliminate this test as long as DECAY > 1. |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 523 | */ |
Tuukka Tikkanen | 51f245b | 2013-08-14 19:02:41 +0300 | [diff] [blame] | 524 | if (DECAY == 1 && unlikely(new_factor == 0)) |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 525 | new_factor = 1; |
venkatesh.pallipadi@intel.com | 320eee7 | 2008-07-30 19:21:43 -0700 | [diff] [blame] | 526 | |
Arjan van de Ven | 69d2587 | 2009-09-21 17:04:08 -0700 | [diff] [blame] | 527 | data->correction_factor[data->bucket] = new_factor; |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 528 | |
| 529 | /* update the repeating-pattern data */ |
Rafael J. Wysocki | c1d51f6 | 2019-11-07 15:25:12 +0100 | [diff] [blame] | 530 | data->intervals[data->interval_ptr++] = ktime_to_us(measured_ns); |
Arjan van de Ven | 1f85f87 | 2010-05-24 14:32:59 -0700 | [diff] [blame] | 531 | if (data->interval_ptr >= INTERVALS) |
| 532 | data->interval_ptr = 0; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 533 | } |
| 534 | |
| 535 | /** |
| 536 | * menu_enable_device - scans a CPU's states and does setup |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 537 | * @drv: cpuidle driver |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 538 | * @dev: the CPU |
| 539 | */ |
Deepthi Dharwar | 46bcfad | 2011-10-28 16:20:42 +0530 | [diff] [blame] | 540 | static int menu_enable_device(struct cpuidle_driver *drv, |
| 541 | struct cpuidle_device *dev) |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 542 | { |
| 543 | struct menu_device *data = &per_cpu(menu_devices, dev->cpu); |
Chander Kashyap | bed4d59 | 2014-04-22 18:08:04 +0530 | [diff] [blame] | 544 | int i; |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 545 | |
| 546 | memset(data, 0, sizeof(struct menu_device)); |
| 547 | |
Chander Kashyap | bed4d59 | 2014-04-22 18:08:04 +0530 | [diff] [blame] | 548 | /* |
| 549 | * if the correction factor is 0 (eg first time init or cpu hotplug |
| 550 | * etc), we actually want to start out with a unity factor. |
| 551 | */ |
| 552 | for(i = 0; i < BUCKETS; i++) |
| 553 | data->correction_factor[i] = RESOLUTION * DECAY; |
| 554 | |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 555 | return 0; |
| 556 | } |
| 557 | |
| 558 | static struct cpuidle_governor menu_governor = { |
| 559 | .name = "menu", |
| 560 | .rating = 20, |
| 561 | .enable = menu_enable_device, |
| 562 | .select = menu_select, |
| 563 | .reflect = menu_reflect, |
Len Brown | 4f86d3a | 2007-10-03 18:58:00 -0400 | [diff] [blame] | 564 | }; |
| 565 | |
| 566 | /** |
| 567 | * init_menu - initializes the governor |
| 568 | */ |
| 569 | static int __init init_menu(void) |
| 570 | { |
| 571 | return cpuidle_register_governor(&menu_governor); |
| 572 | } |
| 573 | |
Daniel Lezcano | 137b944 | 2013-06-12 15:08:48 +0200 | [diff] [blame] | 574 | postcore_initcall(init_menu); |