summaryrefslogtreecommitdiff
path: root/drivers/cpuidle/governors/teo.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/cpuidle/governors/teo.c')
-rw-r--r--drivers/cpuidle/governors/teo.c368
1 files changed, 200 insertions, 168 deletions
diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c
index 173ab30b9a06..5bcd45f1d610 100644
--- a/drivers/cpuidle/governors/teo.c
+++ b/drivers/cpuidle/governors/teo.c
@@ -2,47 +2,90 @@
/*
* Timer events oriented CPU idle governor
*
- * Copyright (C) 2018 Intel Corporation
+ * Copyright (C) 2018 - 2021 Intel Corporation
* Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
*
* The idea of this governor is based on the observation that on many systems
* timer events are two or more orders of magnitude more frequent than any
- * other interrupts, so they are likely to be the most significant source of CPU
+ * other interrupts, so they are likely to be the most significant cause of CPU
* wakeups from idle states. Moreover, information about what happened in the
* (relatively recent) past can be used to estimate whether or not the deepest
- * idle state with target residency within the time to the closest timer is
- * likely to be suitable for the upcoming idle time of the CPU and, if not, then
- * which of the shallower idle states to choose.
+ * idle state with target residency within the (known) time till the closest
+ * timer event, referred to as the sleep length, is likely to be suitable for
+ * the upcoming CPU idle period and, if not, then which of the shallower idle
+ * states to choose instead of it.
*
- * Of course, non-timer wakeup sources are more important in some use cases and
- * they can be covered by taking a few most recent idle time intervals of the
- * CPU into account. However, even in that case it is not necessary to consider
- * idle duration values greater than the time till the closest timer, as the
- * patterns that they may belong to produce average values close enough to
- * the time till the closest timer (sleep length) anyway.
+ * Of course, non-timer wakeup sources are more important in some use cases
+ * which can be covered by taking a few most recent idle time intervals of the
+ * CPU into account. However, even in that context it is not necessary to
+ * consider idle duration values greater than the sleep length, because the
+ * closest timer will ultimately wake up the CPU anyway unless it is woken up
+ * earlier.
*
- * Thus this governor estimates whether or not the upcoming idle time of the CPU
- * is likely to be significantly shorter than the sleep length and selects an
- * idle state for it in accordance with that, as follows:
+ * Thus this governor estimates whether or not the prospective idle duration of
+ * a CPU is likely to be significantly shorter than the sleep length and selects
+ * an idle state for it accordingly.
*
- * - Find an idle state on the basis of the sleep length and state statistics
- * collected over time:
+ * The computations carried out by this governor are based on using bins whose
+ * boundaries are aligned with the target residency parameter values of the CPU
+ * idle states provided by the cpuidle driver in the ascending order. That is,
+ * the first bin spans from 0 up to, but not including, the target residency of
+ * the second idle state (idle state 1), the second bin spans from the target
+ * residency of idle state 1 up to, but not including, the target residency of
+ * idle state 2, the third bin spans from the target residency of idle state 2
+ * up to, but not including, the target residency of idle state 3 and so on.
+ * The last bin spans from the target residency of the deepest idle state
+ * supplied by the driver to infinity.
*
- * o Find the deepest idle state whose target residency is less than or equal
- * to the sleep length.
+ * Two metrics called "hits" and "intercepts" are associated with each bin.
+ * They are updated every time before selecting an idle state for the given CPU
+ * in accordance with what happened last time.
*
- * o Select it if it matched both the sleep length and the observed idle
- * duration in the past more often than it matched the sleep length alone
- * (i.e. the observed idle duration was significantly shorter than the sleep
- * length matched by it).
+ * The "hits" metric reflects the relative frequency of situations in which the
+ * sleep length and the idle duration measured after CPU wakeup fall into the
+ * same bin (that is, the CPU appears to wake up "on time" relative to the sleep
+ * length). In turn, the "intercepts" metric reflects the relative frequency of
+ * situations in which the measured idle duration is so much shorter than the
+ * sleep length that the bin it falls into corresponds to an idle state
+ * shallower than the one whose bin is fallen into by the sleep length.
*
- * o Otherwise, select the shallower state with the greatest matched "early"
- * wakeups metric.
+ * In order to select an idle state for a CPU, the governor takes the following
+ * steps (modulo the possible latency constraint that must be taken into account
+ * too):
*
- * - If the majority of the most recent idle duration values are below the
- * target residency of the idle state selected so far, use those values to
- * compute the new expected idle duration and find an idle state matching it
- * (which has to be shallower than the one selected so far).
+ * 1. Find the deepest CPU idle state whose target residency does not exceed
+ * the current sleep length (the candidate idle state) and compute two sums
+ * as follows:
+ *
+ * - The sum of the "hits" and "intercepts" metrics for the candidate state
+ * and all of the deeper idle states (it represents the cases in which the
+ * CPU was idle long enough to avoid being intercepted if the sleep length
+ * had been equal to the current one).
+ *
+ * - The sum of the "intercepts" metrics for all of the idle states shallower
+ * than the candidate one (it represents the cases in which the CPU was not
+ * idle long enough to avoid being intercepted if the sleep length had been
+ * equal to the current one).
+ *
+ * 2. If the second sum is greater than the first one, look for an alternative
+ * idle state to select.
+ *
+ * - Traverse the idle states shallower than the candidate one in the
+ * descending order.
+ *
+ * - For each of them compute the sum of the "intercepts" metrics over all of
+ * the idle states between it and the candidate one (including the former
+ * and excluding the latter).
+ *
+ * - If that sum is greater than a half of the second sum computed in step 1
+ * (which means that the target residency of the state in question had not
+ * exceeded the idle duration in over a half of the relevant cases), select
+ * the given idle state instead of the candidate one.
+ *
+ * 3. If the majority of the most recent idle duration values are below the
+ * current anticipated idle duration, use those values to compute the new
+ * expected idle duration and find an idle state matching it (which has to
+ * be shallower than the current candidate one).
*/
#include <linux/cpuidle.h>
@@ -65,44 +108,29 @@
#define INTERVALS 8
/**
- * struct teo_idle_state - Idle state data used by the TEO cpuidle governor.
- * @early_hits: "Early" CPU wakeups "matching" this state.
- * @hits: "On time" CPU wakeups "matching" this state.
- * @misses: CPU wakeups "missing" this state.
- *
- * A CPU wakeup is "matched" by a given idle state if the idle duration measured
- * after the wakeup is between the target residency of that state and the target
- * residency of the next one (or if this is the deepest available idle state, it
- * "matches" a CPU wakeup when the measured idle duration is at least equal to
- * its target residency).
- *
- * Also, from the TEO governor perspective, a CPU wakeup from idle is "early" if
- * it occurs significantly earlier than the closest expected timer event (that
- * is, early enough to match an idle state shallower than the one matching the
- * time till the closest timer event). Otherwise, the wakeup is "on time", or
- * it is a "hit".
- *
- * A "miss" occurs when the given state doesn't match the wakeup, but it matches
- * the time till the closest timer event used for idle state selection.
+ * struct teo_bin - Metrics used by the TEO cpuidle governor.
+ * @intercepts: The "intercepts" metric.
+ * @hits: The "hits" metric.
*/
-struct teo_idle_state {
- unsigned int early_hits;
+struct teo_bin {
+ unsigned int intercepts;
unsigned int hits;
- unsigned int misses;
};
/**
* struct teo_cpu - CPU data used by the TEO cpuidle governor.
* @time_span_ns: Time between idle state selection and post-wakeup update.
* @sleep_length_ns: Time till the closest timer event (at the selection time).
- * @states: Idle states data corresponding to this CPU.
+ * @state_bins: Idle state data bins for this CPU.
+ * @total: Grand total of the "intercepts" and "hits" mertics for all bins.
* @interval_idx: Index of the most recent saved idle interval.
* @intervals: Saved idle duration values.
*/
struct teo_cpu {
s64 time_span_ns;
s64 sleep_length_ns;
- struct teo_idle_state states[CPUIDLE_STATE_MAX];
+ struct teo_bin state_bins[CPUIDLE_STATE_MAX];
+ unsigned int total;
int interval_idx;
u64 intervals[INTERVALS];
};
@@ -110,7 +138,7 @@ struct teo_cpu {
static DEFINE_PER_CPU(struct teo_cpu, teo_cpus);
/**
- * teo_update - Update CPU data after wakeup.
+ * teo_update - Update CPU metrics after wakeup.
* @drv: cpuidle driver containing state data.
* @dev: Target CPU.
*/
@@ -118,7 +146,6 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
{
struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
int i, idx_timer = 0, idx_duration = 0;
- unsigned int hits, misses;
u64 measured_ns;
if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) {
@@ -151,15 +178,21 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
measured_ns /= 2;
}
+ cpu_data->total = 0;
+
/*
- * Decay the "early hits" metric for all of the states and find the
- * states matching the sleep length and the measured idle duration.
+ * Decay the "hits" and "intercepts" metrics for all of the bins and
+ * find the bins that the sleep length and the measured idle duration
+ * fall into.
*/
for (i = 0; i < drv->state_count; i++) {
s64 target_residency_ns = drv->states[i].target_residency_ns;
- unsigned int early_hits = cpu_data->states[i].early_hits;
+ struct teo_bin *bin = &cpu_data->state_bins[i];
+
+ bin->hits -= bin->hits >> DECAY_SHIFT;
+ bin->intercepts -= bin->intercepts >> DECAY_SHIFT;
- cpu_data->states[i].early_hits -= early_hits >> DECAY_SHIFT;
+ cpu_data->total += bin->hits + bin->intercepts;
if (target_residency_ns <= cpu_data->sleep_length_ns) {
idx_timer = i;
@@ -169,28 +202,17 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
}
/*
- * Update the "hits" and "misses" data for the state matching the sleep
- * length. If it matches the measured idle duration too, this is a hit,
- * so increase the "hits" metric for it then. Otherwise, this is a
- * miss, so increase the "misses" metric for it. In the latter case
- * also increase the "early hits" metric for the state that actually
- * matches the measured idle duration.
+ * If the measured idle duration falls into the same bin as the sleep
+ * length, this is a "hit", so update the "hits" metric for that bin.
+ * Otherwise, update the "intercepts" metric for the bin fallen into by
+ * the measured idle duration.
*/
- hits = cpu_data->states[idx_timer].hits;
- hits -= hits >> DECAY_SHIFT;
-
- misses = cpu_data->states[idx_timer].misses;
- misses -= misses >> DECAY_SHIFT;
-
- if (idx_timer == idx_duration) {
- hits += PULSE;
- } else {
- misses += PULSE;
- cpu_data->states[idx_duration].early_hits += PULSE;
- }
+ if (idx_timer == idx_duration)
+ cpu_data->state_bins[idx_timer].hits += PULSE;
+ else
+ cpu_data->state_bins[idx_duration].intercepts += PULSE;
- cpu_data->states[idx_timer].misses = misses;
- cpu_data->states[idx_timer].hits = hits;
+ cpu_data->total += PULSE;
/*
* Save idle duration values corresponding to non-timer wakeups for
@@ -206,6 +228,12 @@ static bool teo_time_ok(u64 interval_ns)
return !tick_nohz_tick_stopped() || interval_ns >= TICK_NSEC;
}
+static s64 teo_middle_of_bin(int idx, struct cpuidle_driver *drv)
+{
+ return (drv->states[idx].target_residency_ns +
+ drv->states[idx+1].target_residency_ns) / 2;
+}
+
/**
* teo_find_shallower_state - Find shallower idle state matching given duration.
* @drv: cpuidle driver containing state data.
@@ -241,12 +269,12 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
{
struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
- int constraint_idx = drv->state_count;
- unsigned int hits = 0, misses = 0;
- unsigned int early_hits = 0;
- int prev_max_early_idx = -1;
- int max_early_idx = -1;
- int idx0 = -1, idx = -1;
+ unsigned int idx_intercept_sum = 0;
+ unsigned int intercept_sum = 0;
+ unsigned int idx_hit_sum = 0;
+ unsigned int hit_sum = 0;
+ int constraint_idx = 0;
+ int idx0 = 0, idx = -1;
ktime_t delta_tick;
s64 duration_ns;
int i;
@@ -261,119 +289,122 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
duration_ns = tick_nohz_get_sleep_length(&delta_tick);
cpu_data->sleep_length_ns = duration_ns;
- for (i = 0; i < drv->state_count; i++) {
- struct cpuidle_state *s = &drv->states[i];
-
- if (dev->states_usage[i].disable) {
- /*
- * Ignore disabled states with target residencies beyond
- * the anticipated idle duration.
- */
- if (s->target_residency_ns > duration_ns)
- continue;
-
- /*
- * This state is disabled, so the range of idle duration
- * values corresponding to it is covered by the current
- * candidate state, but still the "hits" and "misses"
- * metrics of the disabled state need to be used to
- * decide whether or not the state covering the range in
- * question is good enough.
- */
- hits = cpu_data->states[i].hits;
- misses = cpu_data->states[i].misses;
-
- if (early_hits >= cpu_data->states[i].early_hits ||
- idx < 0)
- continue;
+ /* Check if there is any choice in the first place. */
+ if (drv->state_count < 2) {
+ idx = 0;;
+ goto end;
+ }
+ if (!dev->states_usage[0].disable) {
+ idx = 0;
+ if (drv->states[1].target_residency_ns > duration_ns)
+ goto end;
+ }
- /*
- * If the current candidate state has been the one with
- * the maximum "early hits" metric so far, the "early
- * hits" metric of the disabled state replaces the
- * current "early hits" count to avoid selecting a
- * deeper state with lower "early hits" metric.
- */
- if (max_early_idx == idx) {
- early_hits = cpu_data->states[i].early_hits;
- continue;
- }
+ /*
+ * Find the deepest idle state whose target residency does not exceed
+ * the current sleep length and the deepest idle state not deeper than
+ * the former whose exit latency does not exceed the current latency
+ * constraint. Compute the sums of metrics for early wakeup pattern
+ * detection.
+ */
+ for (i = 1; i < drv->state_count; i++) {
+ struct teo_bin *prev_bin = &cpu_data->state_bins[i-1];
+ struct cpuidle_state *s = &drv->states[i];
- /*
- * The current candidate state is closer to the disabled
- * one than the current maximum "early hits" state, so
- * replace the latter with it, but in case the maximum
- * "early hits" state index has not been set so far,
- * check if the current candidate state is not too
- * shallow for that role.
- */
- if (teo_time_ok(drv->states[idx].target_residency_ns)) {
- prev_max_early_idx = max_early_idx;
- early_hits = cpu_data->states[i].early_hits;
- max_early_idx = idx;
- }
+ /*
+ * Update the sums of idle state mertics for all of the states
+ * shallower than the current one.
+ */
+ intercept_sum += prev_bin->intercepts;
+ hit_sum += prev_bin->hits;
+ if (dev->states_usage[i].disable)
continue;
- }
if (idx < 0) {
idx = i; /* first enabled state */
- hits = cpu_data->states[i].hits;
- misses = cpu_data->states[i].misses;
idx0 = i;
}
if (s->target_residency_ns > duration_ns)
break;
- if (s->exit_latency_ns > latency_req && constraint_idx > i)
+ idx = i;
+
+ if (s->exit_latency_ns <= latency_req)
constraint_idx = i;
- idx = i;
- hits = cpu_data->states[i].hits;
- misses = cpu_data->states[i].misses;
-
- if (early_hits < cpu_data->states[i].early_hits &&
- teo_time_ok(drv->states[i].target_residency_ns)) {
- prev_max_early_idx = max_early_idx;
- early_hits = cpu_data->states[i].early_hits;
- max_early_idx = i;
- }
+ idx_intercept_sum = intercept_sum;
+ idx_hit_sum = hit_sum;
+ }
+
+ /* Avoid unnecessary overhead. */
+ if (idx < 0) {
+ idx = 0; /* No states enabled, must use 0. */
+ goto end;
+ } else if (idx == idx0) {
+ goto end;
}
/*
- * If the "hits" metric of the idle state matching the sleep length is
- * greater than its "misses" metric, that is the one to use. Otherwise,
- * it is more likely that one of the shallower states will match the
- * idle duration observed after wakeup, so take the one with the maximum
- * "early hits" metric, but if that cannot be determined, just use the
- * state selected so far.
+ * If the sum of the intercepts metric for all of the idle states
+ * shallower than the current candidate one (idx) is greater than the
+ * sum of the intercepts and hits metrics for the candidate state and
+ * all of the deeper states, the CPU is likely to wake up early, so find
+ * an alternative idle state to select.
*/
- if (hits <= misses) {
+ if (2 * idx_intercept_sum > cpu_data->total - idx_hit_sum) {
+ s64 last_enabled_span_ns = duration_ns;
+ int last_enabled_idx = idx;
+
/*
- * The current candidate state is not suitable, so take the one
- * whose "early hits" metric is the maximum for the range of
- * shallower states.
+ * Look for the deepest idle state whose target residency had
+ * not exceeded the idle duration in over a half of the relevant
+ * cases in the past.
+ *
+ * Take the possible latency constraint and duration limitation
+ * present if the tick has been stopped already into account.
*/
- if (idx == max_early_idx)
- max_early_idx = prev_max_early_idx;
+ intercept_sum = 0;
+
+ for (i = idx - 1; i >= idx0; i--) {
+ s64 span_ns;
- if (max_early_idx >= 0) {
- idx = max_early_idx;
- duration_ns = drv->states[idx].target_residency_ns;
+ intercept_sum += cpu_data->state_bins[i].intercepts;
+
+ if (dev->states_usage[i].disable)
+ continue;
+
+ span_ns = teo_middle_of_bin(i, drv);
+ if (!teo_time_ok(span_ns)) {
+ /*
+ * The current state is too shallow, so select
+ * the first enabled deeper state.
+ */
+ duration_ns = last_enabled_span_ns;
+ idx = last_enabled_idx;
+ break;
+ }
+
+ if (2 * intercept_sum > idx_intercept_sum) {
+ idx = i;
+ duration_ns = span_ns;
+ break;
+ }
+
+ last_enabled_span_ns = span_ns;
+ last_enabled_idx = i;
}
}
/*
- * If there is a latency constraint, it may be necessary to use a
- * shallower idle state than the one selected so far.
+ * If there is a latency constraint, it may be necessary to select an
+ * idle state shallower than the current candidate one.
*/
- if (constraint_idx < idx)
+ if (idx > constraint_idx)
idx = constraint_idx;
- if (idx < 0) {
- idx = 0; /* No states enabled. Must use 0. */
- } else if (idx > idx0) {
+ if (idx > idx0) {
unsigned int count = 0;
u64 sum = 0;
@@ -416,6 +447,7 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
}
}
+end:
/*
* Don't stop the tick if the selected state is a polling one or if the
* expected idle duration is shorter than the tick period length.
generated by cgit v1.2.3 (git 2.39.0) at 2024-07-23 00:07:53 +0300