Update Linux to v5.10.109
Sourced from [1]
[1] https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.109.tar.xz
Change-Id: I19bca9fc6762d4e63bcf3e4cba88bbe560d9c76c
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 88fe803..1686705 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -36,6 +36,7 @@
#define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
+#define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
#define INTEL_CPUFREQ_TRANSITION_DELAY 500
#ifdef CONFIG_ACPI
@@ -172,7 +173,7 @@
/**
* struct global_params - Global parameters, mostly tunable via sysfs.
* @no_turbo: Whether or not to use turbo P-states.
- * @turbo_disabled: Whethet or not turbo P-states are available at all,
+ * @turbo_disabled: Whether or not turbo P-states are available at all,
* based on the MSR_IA32_MISC_ENABLE value and whether or
* not the maximum reported turbo P-state is different from
* the maximum reported non-turbo one.
@@ -201,9 +202,7 @@
* @pstate: Stores P state limits for this CPU
* @vid: Stores VID limits for this CPU
* @last_sample_time: Last Sample time
- * @aperf_mperf_shift: Number of clock cycles after aperf, merf is incremented
- * This shift is a multiplier to mperf delta to
- * calculate CPU busy.
+ * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
* @prev_aperf: Last APERF value read from APERF MSR
* @prev_mperf: Last MPERF value read from MPERF MSR
* @prev_tsc: Last timestamp counter (TSC) value
@@ -220,13 +219,13 @@
* @epp_policy: Last saved policy used to set EPP/EPB
* @epp_default: Power on default HWP energy performance
* preference/bias
- * @epp_saved: Saved EPP/EPB during system suspend or CPU offline
- * operation
+ * @epp_cached Cached HWP energy-performance preference value
* @hwp_req_cached: Cached value of the last HWP Request MSR
* @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
* @last_io_update: Last time when IO wake flag was set
* @sched_flags: Store scheduler flags for possible cross CPU update
* @hwp_boost_min: Last HWP boosted min performance
+ * @suspended: Whether or not the driver has been suspended.
*
* This structure stores per CPU instance data for all CPUs.
*/
@@ -258,12 +257,13 @@
s16 epp_powersave;
s16 epp_policy;
s16 epp_default;
- s16 epp_saved;
+ s16 epp_cached;
u64 hwp_req_cached;
u64 hwp_cap_cached;
u64 last_io_update;
unsigned int sched_flags;
u32 hwp_boost_min;
+ bool suspended;
};
static struct cpudata **all_cpu_data;
@@ -275,6 +275,7 @@
* @get_min: Callback to get minimum P state
* @get_turbo: Callback to get turbo P state
* @get_scaling: Callback to get frequency scaling factor
+ * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
* @get_val: Callback to convert P state to actual MSR write value
* @get_vid: Callback to get VID data for Atom platforms
*
@@ -602,11 +603,12 @@
HWP_EPP_POWERSAVE
};
-static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data)
+static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)
{
s16 epp;
int index = -EINVAL;
+ *raw_epp = 0;
epp = intel_pstate_get_epp(cpu_data, 0);
if (epp < 0)
return epp;
@@ -614,12 +616,14 @@
if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
if (epp == HWP_EPP_PERFORMANCE)
return 1;
- if (epp <= HWP_EPP_BALANCE_PERFORMANCE)
+ if (epp == HWP_EPP_BALANCE_PERFORMANCE)
return 2;
- if (epp <= HWP_EPP_BALANCE_POWERSAVE)
+ if (epp == HWP_EPP_BALANCE_POWERSAVE)
return 3;
- else
+ if (epp == HWP_EPP_POWERSAVE)
return 4;
+ *raw_epp = epp;
+ return 0;
} else if (boot_cpu_has(X86_FEATURE_EPB)) {
/*
* Range:
@@ -637,8 +641,35 @@
return index;
}
+static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)
+{
+ int ret;
+
+ /*
+ * Use the cached HWP Request MSR value, because in the active mode the
+ * register itself may be updated by intel_pstate_hwp_boost_up() or
+ * intel_pstate_hwp_boost_down() at any time.
+ */
+ u64 value = READ_ONCE(cpu->hwp_req_cached);
+
+ value &= ~GENMASK_ULL(31, 24);
+ value |= (u64)epp << 24;
+ /*
+ * The only other updater of hwp_req_cached in the active mode,
+ * intel_pstate_hwp_set(), is called under the same lock as this
+ * function, so it cannot run in parallel with the update below.
+ */
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ ret = wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
+ if (!ret)
+ cpu->epp_cached = epp;
+
+ return ret;
+}
+
static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
- int pref_index)
+ int pref_index, bool use_raw,
+ u32 raw_epp)
{
int epp = -EINVAL;
int ret;
@@ -646,35 +677,26 @@
if (!pref_index)
epp = cpu_data->epp_default;
- mutex_lock(&intel_pstate_limits_lock);
-
if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
- /*
- * Use the cached HWP Request MSR value, because the register
- * itself may be updated by intel_pstate_hwp_boost_up() or
- * intel_pstate_hwp_boost_down() at any time.
- */
- u64 value = READ_ONCE(cpu_data->hwp_req_cached);
-
- value &= ~GENMASK_ULL(31, 24);
-
- if (epp == -EINVAL)
+ if (use_raw)
+ epp = raw_epp;
+ else if (epp == -EINVAL)
epp = epp_values[pref_index - 1];
- value |= (u64)epp << 24;
/*
- * The only other updater of hwp_req_cached in the active mode,
- * intel_pstate_hwp_set(), is called under the same lock as this
- * function, so it cannot run in parallel with the update below.
+ * To avoid confusion, refuse to set EPP to any values different
+ * from 0 (performance) if the current policy is "performance",
+ * because those values would be overridden.
*/
- WRITE_ONCE(cpu_data->hwp_req_cached, value);
- ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value);
+ if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
+ return -EBUSY;
+
+ ret = intel_pstate_set_epp(cpu_data, epp);
} else {
if (epp == -EINVAL)
epp = (pref_index - 1) << 2;
ret = intel_pstate_set_epb(cpu_data->cpu, epp);
}
- mutex_unlock(&intel_pstate_limits_lock);
return ret;
}
@@ -695,36 +717,88 @@
cpufreq_freq_attr_ro(energy_performance_available_preferences);
+static struct cpufreq_driver intel_pstate;
+
static ssize_t store_energy_performance_preference(
struct cpufreq_policy *policy, const char *buf, size_t count)
{
- struct cpudata *cpu_data = all_cpu_data[policy->cpu];
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
char str_preference[21];
- int ret;
+ bool raw = false;
+ ssize_t ret;
+ u32 epp = 0;
ret = sscanf(buf, "%20s", str_preference);
if (ret != 1)
return -EINVAL;
ret = match_string(energy_perf_strings, -1, str_preference);
- if (ret < 0)
- return ret;
+ if (ret < 0) {
+ if (!boot_cpu_has(X86_FEATURE_HWP_EPP))
+ return ret;
- intel_pstate_set_energy_pref_index(cpu_data, ret);
- return count;
+ ret = kstrtouint(buf, 10, &epp);
+ if (ret)
+ return ret;
+
+ if (epp > 255)
+ return -EINVAL;
+
+ raw = true;
+ }
+
+ /*
+ * This function runs with the policy R/W semaphore held, which
+ * guarantees that the driver pointer will not change while it is
+ * running.
+ */
+ if (!intel_pstate_driver)
+ return -EAGAIN;
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ if (intel_pstate_driver == &intel_pstate) {
+ ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp);
+ } else {
+ /*
+ * In the passive mode the governor needs to be stopped on the
+ * target CPU before the EPP update and restarted after it,
+ * which is super-heavy-weight, so make sure it is worth doing
+ * upfront.
+ */
+ if (!raw)
+ epp = ret ? epp_values[ret - 1] : cpu->epp_default;
+
+ if (cpu->epp_cached != epp) {
+ int err;
+
+ cpufreq_stop_governor(policy);
+ ret = intel_pstate_set_epp(cpu, epp);
+ err = cpufreq_start_governor(policy);
+ if (!ret)
+ ret = err;
+ }
+ }
+
+ mutex_unlock(&intel_pstate_limits_lock);
+
+ return ret ?: count;
}
static ssize_t show_energy_performance_preference(
struct cpufreq_policy *policy, char *buf)
{
struct cpudata *cpu_data = all_cpu_data[policy->cpu];
- int preference;
+ int preference, raw_epp;
- preference = intel_pstate_get_energy_pref_index(cpu_data);
+ preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp);
if (preference < 0)
return preference;
- return sprintf(buf, "%s\n", energy_perf_strings[preference]);
+ if (raw_epp)
+ return sprintf(buf, "%d\n", raw_epp);
+ else
+ return sprintf(buf, "%s\n", energy_perf_strings[preference]);
}
cpufreq_freq_attr_rw(energy_performance_preference);
@@ -755,13 +829,13 @@
NULL,
};
-static void intel_pstate_get_hwp_max(unsigned int cpu, int *phy_max,
+static void intel_pstate_get_hwp_max(struct cpudata *cpu, int *phy_max,
int *current_max)
{
u64 cap;
- rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
- WRITE_ONCE(all_cpu_data[cpu]->hwp_cap_cached, cap);
+ rdmsrl_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap);
+ WRITE_ONCE(cpu->hwp_cap_cached, cap);
if (global.no_turbo || global.turbo_disabled)
*current_max = HWP_GUARANTEED_PERF(cap);
else
@@ -796,12 +870,6 @@
cpu_data->epp_policy = cpu_data->policy;
- if (cpu_data->epp_saved >= 0) {
- epp = cpu_data->epp_saved;
- cpu_data->epp_saved = -EINVAL;
- goto update_epp;
- }
-
if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
epp = intel_pstate_get_epp(cpu_data, value);
cpu_data->epp_powersave = epp;
@@ -828,7 +896,6 @@
epp = cpu_data->epp_powersave;
}
-update_epp:
if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
value &= ~GENMASK_ULL(31, 24);
value |= (u64)epp << 24;
@@ -840,14 +907,24 @@
wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
}
-static void intel_pstate_hwp_force_min_perf(int cpu)
+static void intel_pstate_hwp_offline(struct cpudata *cpu)
{
- u64 value;
+ u64 value = READ_ONCE(cpu->hwp_req_cached);
int min_perf;
- value = all_cpu_data[cpu]->hwp_req_cached;
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ /*
+ * In case the EPP has been set to "performance" by the
+ * active mode "performance" scaling algorithm, replace that
+ * temporary value with the cached EPP one.
+ */
+ value &= ~GENMASK_ULL(31, 24);
+ value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ }
+
value &= ~GENMASK_ULL(31, 0);
- min_perf = HWP_LOWEST_PERF(all_cpu_data[cpu]->hwp_cap_cached);
+ min_perf = HWP_LOWEST_PERF(cpu->hwp_cap_cached);
/* Set hwp_max = hwp_min */
value |= HWP_MAX_PERF(min_perf);
@@ -857,37 +934,72 @@
if (boot_cpu_has(X86_FEATURE_HWP_EPP))
value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE);
- wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
}
-static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
+#define POWER_CTL_EE_ENABLE 1
+#define POWER_CTL_EE_DISABLE 2
+
+static int power_ctl_ee_state;
+
+static void set_power_ctl_ee_state(bool input)
{
- struct cpudata *cpu_data = all_cpu_data[policy->cpu];
+ u64 power_ctl;
- if (!hwp_active)
- return 0;
-
- cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0);
-
- return 0;
+ mutex_lock(&intel_pstate_driver_lock);
+ rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
+ if (input) {
+ power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE);
+ power_ctl_ee_state = POWER_CTL_EE_ENABLE;
+ } else {
+ power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
+ power_ctl_ee_state = POWER_CTL_EE_DISABLE;
+ }
+ wrmsrl(MSR_IA32_POWER_CTL, power_ctl);
+ mutex_unlock(&intel_pstate_driver_lock);
}
static void intel_pstate_hwp_enable(struct cpudata *cpudata);
+static void intel_pstate_hwp_reenable(struct cpudata *cpu)
+{
+ intel_pstate_hwp_enable(cpu);
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached));
+}
+
+static int intel_pstate_suspend(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d suspending\n", cpu->cpu);
+
+ cpu->suspended = true;
+
+ return 0;
+}
+
static int intel_pstate_resume(struct cpufreq_policy *policy)
{
- if (!hwp_active)
- return 0;
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
- mutex_lock(&intel_pstate_limits_lock);
+ pr_debug("CPU %d resuming\n", cpu->cpu);
- if (policy->cpu == 0)
- intel_pstate_hwp_enable(all_cpu_data[policy->cpu]);
+ /* Only restore if the system default is changed */
+ if (power_ctl_ee_state == POWER_CTL_EE_ENABLE)
+ set_power_ctl_ee_state(true);
+ else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE)
+ set_power_ctl_ee_state(false);
- all_cpu_data[policy->cpu]->epp_policy = 0;
- intel_pstate_hwp_set(policy->cpu);
+ if (cpu->suspended && hwp_active) {
+ mutex_lock(&intel_pstate_limits_lock);
- mutex_unlock(&intel_pstate_limits_lock);
+ /* Re-enable HWP, because "online" has not done that. */
+ intel_pstate_hwp_reenable(cpu);
+
+ mutex_unlock(&intel_pstate_limits_lock);
+ }
+
+ cpu->suspended = false;
return 0;
}
@@ -928,6 +1040,7 @@
*/
if (global.turbo_disabled_mf != global.turbo_disabled) {
global.turbo_disabled_mf = global.turbo_disabled;
+ arch_set_max_freq_ratio(global.turbo_disabled);
for_each_possible_cpu(cpu)
intel_pstate_update_max_freq(cpu);
} else {
@@ -1090,8 +1203,6 @@
return count;
}
-static struct cpufreq_driver intel_pstate;
-
static void update_qos_request(enum freq_qos_req_type type)
{
int max_state, turbo_max, freq, i, perf_pct;
@@ -1112,7 +1223,7 @@
continue;
if (hwp_active)
- intel_pstate_get_hwp_max(i, &turbo_max, &max_state);
+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
else
turbo_max = cpu->pstate.turbo_pstate;
@@ -1223,6 +1334,32 @@
return count;
}
+static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ u64 power_ctl;
+ int enable;
+
+ rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
+ enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE));
+ return sprintf(buf, "%d\n", !enable);
+}
+
+static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ bool input;
+ int ret;
+
+ ret = kstrtobool(buf, &input);
+ if (ret)
+ return ret;
+
+ set_power_ctl_ee_state(input);
+
+ return count;
+}
+
show_one(max_perf_pct, max_perf_pct);
show_one(min_perf_pct, min_perf_pct);
@@ -1233,6 +1370,7 @@
define_one_global_ro(turbo_pct);
define_one_global_ro(num_pstates);
define_one_global_rw(hwp_dynamic_boost);
+define_one_global_rw(energy_efficiency);
static struct attribute *intel_pstate_attributes[] = {
&status.attr,
@@ -1246,9 +1384,12 @@
.attrs = intel_pstate_attributes,
};
+static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[];
+
+static struct kobject *intel_pstate_kobject;
+
static void __init intel_pstate_sysfs_expose_params(void)
{
- struct kobject *intel_pstate_kobject;
int rc;
intel_pstate_kobject = kobject_create_and_add("intel_pstate",
@@ -1273,12 +1414,49 @@
rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
WARN_ON(rc);
- if (hwp_active) {
- rc = sysfs_create_file(intel_pstate_kobject,
- &hwp_dynamic_boost.attr);
+ if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) {
+ rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr);
WARN_ON(rc);
}
}
+
+static void __init intel_pstate_sysfs_remove(void)
+{
+ if (!intel_pstate_kobject)
+ return;
+
+ sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
+
+ if (!per_cpu_limits) {
+ sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
+ sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
+
+ if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids))
+ sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr);
+ }
+
+ kobject_put(intel_pstate_kobject);
+}
+
+static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
+{
+ int rc;
+
+ if (!hwp_active)
+ return;
+
+ rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
+ WARN_ON_ONCE(rc);
+}
+
+static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
+{
+ if (!hwp_active)
+ return;
+
+ sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
+}
+
/************************** sysfs end ************************/
static void intel_pstate_hwp_enable(struct cpudata *cpudata)
@@ -1288,30 +1466,10 @@
wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
- cpudata->epp_policy = 0;
if (cpudata->epp_default == -EINVAL)
cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
}
-#define MSR_IA32_POWER_CTL_BIT_EE 19
-
-/* Disable energy efficiency optimization */
-static void intel_pstate_disable_ee(int cpu)
-{
- u64 power_ctl;
- int ret;
-
- ret = rdmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, &power_ctl);
- if (ret)
- return;
-
- if (!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE))) {
- pr_info("Disabling energy efficiency optimization\n");
- power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
- wrmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, power_ctl);
- }
-}
-
static int atom_get_min_pstate(void)
{
u64 value;
@@ -1573,7 +1731,7 @@
if (hwp_active && !hwp_mode_bdw) {
unsigned int phy_max, current_max;
- intel_pstate_get_hwp_max(cpu->cpu, &phy_max, ¤t_max);
+ intel_pstate_get_hwp_max(cpu, &phy_max, ¤t_max);
cpu->pstate.turbo_freq = phy_max * cpu->pstate.scaling;
cpu->pstate.turbo_pstate = phy_max;
cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(READ_ONCE(cpu->hwp_cap_cached));
@@ -1916,51 +2074,51 @@
.get_val = core_get_val,
};
-#define ICPU(model, policy) \
- { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
- (unsigned long)&policy }
+#define X86_MATCH(model, policy) \
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
+ X86_FEATURE_APERFMPERF, &policy)
static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
- ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
- ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs),
- ICPU(INTEL_FAM6_ATOM_SILVERMONT, silvermont_funcs),
- ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs),
- ICPU(INTEL_FAM6_HASWELL, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL, core_funcs),
- ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs),
- ICPU(INTEL_FAM6_HASWELL_X, core_funcs),
- ICPU(INTEL_FAM6_HASWELL_L, core_funcs),
- ICPU(INTEL_FAM6_HASWELL_G, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_G, core_funcs),
- ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs),
- ICPU(INTEL_FAM6_SKYLAKE_L, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
- ICPU(INTEL_FAM6_SKYLAKE, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_D, core_funcs),
- ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs),
- ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs),
- ICPU(INTEL_FAM6_ATOM_GOLDMONT, core_funcs),
- ICPU(INTEL_FAM6_ATOM_GOLDMONT_PLUS, core_funcs),
- ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
+ X86_MATCH(SANDYBRIDGE, core_funcs),
+ X86_MATCH(SANDYBRIDGE_X, core_funcs),
+ X86_MATCH(ATOM_SILVERMONT, silvermont_funcs),
+ X86_MATCH(IVYBRIDGE, core_funcs),
+ X86_MATCH(HASWELL, core_funcs),
+ X86_MATCH(BROADWELL, core_funcs),
+ X86_MATCH(IVYBRIDGE_X, core_funcs),
+ X86_MATCH(HASWELL_X, core_funcs),
+ X86_MATCH(HASWELL_L, core_funcs),
+ X86_MATCH(HASWELL_G, core_funcs),
+ X86_MATCH(BROADWELL_G, core_funcs),
+ X86_MATCH(ATOM_AIRMONT, airmont_funcs),
+ X86_MATCH(SKYLAKE_L, core_funcs),
+ X86_MATCH(BROADWELL_X, core_funcs),
+ X86_MATCH(SKYLAKE, core_funcs),
+ X86_MATCH(BROADWELL_D, core_funcs),
+ X86_MATCH(XEON_PHI_KNL, knl_funcs),
+ X86_MATCH(XEON_PHI_KNM, knl_funcs),
+ X86_MATCH(ATOM_GOLDMONT, core_funcs),
+ X86_MATCH(ATOM_GOLDMONT_PLUS, core_funcs),
+ X86_MATCH(SKYLAKE_X, core_funcs),
{}
};
MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
- ICPU(INTEL_FAM6_BROADWELL_D, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
- ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
+ X86_MATCH(BROADWELL_D, core_funcs),
+ X86_MATCH(BROADWELL_X, core_funcs),
+ X86_MATCH(SKYLAKE_X, core_funcs),
{}
};
static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
- ICPU(INTEL_FAM6_KABYLAKE, core_funcs),
+ X86_MATCH(KABYLAKE, core_funcs),
{}
};
static const struct x86_cpu_id intel_pstate_hwp_boost_ids[] = {
- ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
- ICPU(INTEL_FAM6_SKYLAKE, core_funcs),
+ X86_MATCH(SKYLAKE_X, core_funcs),
+ X86_MATCH(SKYLAKE, core_funcs),
{}
};
@@ -1977,28 +2135,30 @@
all_cpu_data[cpunum] = cpu;
+ cpu->cpu = cpunum;
+
cpu->epp_default = -EINVAL;
- cpu->epp_powersave = -EINVAL;
- cpu->epp_saved = -EINVAL;
+
+ if (hwp_active) {
+ const struct x86_cpu_id *id;
+
+ intel_pstate_hwp_enable(cpu);
+
+ id = x86_match_cpu(intel_pstate_hwp_boost_ids);
+ if (id && intel_pstate_acpi_pm_profile_server())
+ hwp_boost = true;
+ }
+ } else if (hwp_active) {
+ /*
+ * Re-enable HWP in case this happens after a resume from ACPI
+ * S3 if the CPU was offline during the whole system/resume
+ * cycle.
+ */
+ intel_pstate_hwp_reenable(cpu);
}
- cpu = all_cpu_data[cpunum];
-
- cpu->cpu = cpunum;
-
- if (hwp_active) {
- const struct x86_cpu_id *id;
-
- id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
- if (id)
- intel_pstate_disable_ee(cpunum);
-
- intel_pstate_hwp_enable(cpu);
-
- id = x86_match_cpu(intel_pstate_hwp_boost_ids);
- if (id && intel_pstate_acpi_pm_profile_server())
- hwp_boost = true;
- }
+ cpu->epp_powersave = -EINVAL;
+ cpu->epp_policy = 0;
intel_pstate_get_cpu_pstates(cpu);
@@ -2048,9 +2208,9 @@
unsigned int policy_min,
unsigned int policy_max)
{
- int max_freq = intel_pstate_get_max_freq(cpu);
int32_t max_policy_perf, min_policy_perf;
int max_state, turbo_max;
+ int max_freq;
/*
* HWP needs some special consideration, because on BDX the
@@ -2058,12 +2218,13 @@
* rather than pure ratios.
*/
if (hwp_active) {
- intel_pstate_get_hwp_max(cpu->cpu, &turbo_max, &max_state);
+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
} else {
max_state = global.no_turbo || global.turbo_disabled ?
cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
turbo_max = cpu->pstate.turbo_pstate;
}
+ max_freq = max_state * cpu->pstate.scaling;
max_policy_perf = max_state * policy_max / max_freq;
if (policy_max == policy_min) {
@@ -2163,40 +2324,87 @@
}
}
-static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
+static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,
+ struct cpufreq_policy_data *policy)
{
- struct cpudata *cpu = all_cpu_data[policy->cpu];
+ int max_freq;
update_turbo_state();
- cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
- intel_pstate_get_max_freq(cpu));
+ if (hwp_active) {
+ int max_state, turbo_max;
+
+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
+ max_freq = max_state * cpu->pstate.scaling;
+ } else {
+ max_freq = intel_pstate_get_max_freq(cpu);
+ }
+ cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq);
intel_pstate_adjust_policy_max(cpu, policy);
+}
+
+static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
+{
+ intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy);
return 0;
}
-static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy)
+static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
{
- intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]);
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d going offline\n", cpu->cpu);
+
+ if (cpu->suspended)
+ return 0;
+
+ /*
+ * If the CPU is an SMT thread and it goes offline with the performance
+ * settings different from the minimum, it will prevent its sibling
+ * from getting to lower performance levels, so force the minimum
+ * performance on CPU offline to prevent that from happening.
+ */
+ if (hwp_active)
+ intel_pstate_hwp_offline(cpu);
+ else
+ intel_pstate_set_min_pstate(cpu);
+
+ intel_pstate_exit_perf_limits(policy);
+
+ return 0;
+}
+
+static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d going online\n", cpu->cpu);
+
+ intel_pstate_init_acpi_perf_limits(policy);
+
+ if (hwp_active) {
+ /*
+ * Re-enable HWP and clear the "suspended" flag to let "resume"
+ * know that it need not do that.
+ */
+ intel_pstate_hwp_reenable(cpu);
+ cpu->suspended = false;
+ }
+
+ return 0;
}
static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
{
- pr_debug("CPU %d exiting\n", policy->cpu);
+ pr_debug("CPU %d stopping\n", policy->cpu);
intel_pstate_clear_update_util_hook(policy->cpu);
- if (hwp_active) {
- intel_pstate_hwp_save_state(policy);
- intel_pstate_hwp_force_min_perf(policy->cpu);
- } else {
- intel_cpufreq_stop_cpu(policy);
- }
}
static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
{
- intel_pstate_exit_perf_limits(policy);
+ pr_debug("CPU %d exiting\n", policy->cpu);
policy->fast_switch_possible = false;
@@ -2251,10 +2459,17 @@
if (ret)
return ret;
- if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE))
- policy->policy = CPUFREQ_POLICY_PERFORMANCE;
- else
- policy->policy = CPUFREQ_POLICY_POWERSAVE;
+ /*
+ * Set the policy to powersave to provide a valid fallback value in case
+ * the default cpufreq governor is neither powersave nor performance.
+ */
+ policy->policy = CPUFREQ_POLICY_POWERSAVE;
+
+ if (hwp_active) {
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
+ }
return 0;
}
@@ -2263,11 +2478,13 @@
.flags = CPUFREQ_CONST_LOOPS,
.verify = intel_pstate_verify_policy,
.setpolicy = intel_pstate_set_policy,
- .suspend = intel_pstate_hwp_save_state,
+ .suspend = intel_pstate_suspend,
.resume = intel_pstate_resume,
.init = intel_pstate_cpu_init,
.exit = intel_pstate_cpu_exit,
.stop_cpu = intel_pstate_stop_cpu,
+ .offline = intel_pstate_cpu_offline,
+ .online = intel_pstate_cpu_online,
.update_limits = intel_pstate_update_limits,
.name = "intel_pstate",
};
@@ -2276,12 +2493,7 @@
{
struct cpudata *cpu = all_cpu_data[policy->cpu];
- update_turbo_state();
- cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
- intel_pstate_get_max_freq(cpu));
-
- intel_pstate_adjust_policy_max(cpu, policy);
-
+ intel_pstate_verify_cpu_policy(cpu, policy);
intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
return 0;
@@ -2325,13 +2537,71 @@
fp_toint(cpu->iowait_boost * 100));
}
+static void intel_cpufreq_adjust_hwp(struct cpudata *cpu, u32 target_pstate,
+ bool strict, bool fast_switch)
+{
+ u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev;
+
+ value &= ~HWP_MIN_PERF(~0L);
+ value |= HWP_MIN_PERF(target_pstate);
+
+ /*
+ * The entire MSR needs to be updated in order to update the HWP min
+ * field in it, so opportunistically update the max too if needed.
+ */
+ value &= ~HWP_MAX_PERF(~0L);
+ value |= HWP_MAX_PERF(strict ? target_pstate : cpu->max_perf_ratio);
+
+ if (value == prev)
+ return;
+
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ if (fast_switch)
+ wrmsrl(MSR_HWP_REQUEST, value);
+ else
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
+}
+
+static void intel_cpufreq_adjust_perf_ctl(struct cpudata *cpu,
+ u32 target_pstate, bool fast_switch)
+{
+ if (fast_switch)
+ wrmsrl(MSR_IA32_PERF_CTL,
+ pstate_funcs.get_val(cpu, target_pstate));
+ else
+ wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
+ pstate_funcs.get_val(cpu, target_pstate));
+}
+
+static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy,
+ int target_pstate, bool fast_switch)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ int old_pstate = cpu->pstate.current_pstate;
+
+ target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+ if (hwp_active) {
+ intel_cpufreq_adjust_hwp(cpu, target_pstate,
+ policy->strict_target, fast_switch);
+ cpu->pstate.current_pstate = target_pstate;
+ } else if (target_pstate != old_pstate) {
+ intel_cpufreq_adjust_perf_ctl(cpu, target_pstate, fast_switch);
+ cpu->pstate.current_pstate = target_pstate;
+ }
+
+ intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH :
+ INTEL_PSTATE_TRACE_TARGET, old_pstate);
+
+ return target_pstate;
+}
+
static int intel_cpufreq_target(struct cpufreq_policy *policy,
unsigned int target_freq,
unsigned int relation)
{
struct cpudata *cpu = all_cpu_data[policy->cpu];
struct cpufreq_freqs freqs;
- int target_pstate, old_pstate;
+ int target_pstate;
update_turbo_state();
@@ -2339,6 +2609,7 @@
freqs.new = target_freq;
cpufreq_freq_transition_begin(policy, &freqs);
+
switch (relation) {
case CPUFREQ_RELATION_L:
target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
@@ -2350,15 +2621,11 @@
target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
break;
}
- target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
- old_pstate = cpu->pstate.current_pstate;
- if (target_pstate != cpu->pstate.current_pstate) {
- cpu->pstate.current_pstate = target_pstate;
- wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
- pstate_funcs.get_val(cpu, target_pstate));
- }
+
+ target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false);
+
freqs.new = target_pstate * cpu->pstate.scaling;
- intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_TARGET, old_pstate);
+
cpufreq_freq_transition_end(policy, &freqs, false);
return 0;
@@ -2368,15 +2635,14 @@
unsigned int target_freq)
{
struct cpudata *cpu = all_cpu_data[policy->cpu];
- int target_pstate, old_pstate;
+ int target_pstate;
update_turbo_state();
target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
- target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
- old_pstate = cpu->pstate.current_pstate;
- intel_pstate_update_pstate(cpu, target_pstate);
- intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
+
+ target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true);
+
return target_pstate * cpu->pstate.scaling;
}
@@ -2396,7 +2662,6 @@
return ret;
policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
- policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
/* This reflects the intel_pstate_get_cpu_pstates() setting. */
policy->cur = policy->cpuinfo.min_freq;
@@ -2408,10 +2673,18 @@
cpu = all_cpu_data[policy->cpu];
- if (hwp_active)
- intel_pstate_get_hwp_max(policy->cpu, &turbo_max, &max_state);
- else
+ if (hwp_active) {
+ u64 value;
+
+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
+ policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP;
+ rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value);
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ cpu->epp_cached = intel_pstate_get_epp(cpu, value);
+ } else {
turbo_max = cpu->pstate.turbo_pstate;
+ policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
+ }
min_freq = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
min_freq *= cpu->pstate.scaling;
@@ -2466,12 +2739,15 @@
.fast_switch = intel_cpufreq_fast_switch,
.init = intel_cpufreq_cpu_init,
.exit = intel_cpufreq_cpu_exit,
- .stop_cpu = intel_cpufreq_stop_cpu,
+ .offline = intel_pstate_cpu_offline,
+ .online = intel_pstate_cpu_online,
+ .suspend = intel_pstate_suspend,
+ .resume = intel_pstate_resume,
.update_limits = intel_pstate_update_limits,
.name = "intel_cpufreq",
};
-static struct cpufreq_driver *default_driver = &intel_pstate;
+static struct cpufreq_driver *default_driver;
static void intel_pstate_driver_cleanup(void)
{
@@ -2488,6 +2764,7 @@
}
}
put_online_cpus();
+
intel_pstate_driver = NULL;
}
@@ -2495,6 +2772,9 @@
{
int ret;
+ if (driver == &intel_pstate)
+ intel_pstate_sysfs_expose_hwp_dynamic_boost();
+
memset(&global, 0, sizeof(global));
global.max_perf_pct = 100;
@@ -2510,17 +2790,6 @@
return 0;
}
-static int intel_pstate_unregister_driver(void)
-{
- if (hwp_active)
- return -EBUSY;
-
- cpufreq_unregister_driver(intel_pstate_driver);
- intel_pstate_driver_cleanup();
-
- return 0;
-}
-
static ssize_t intel_pstate_show_status(char *buf)
{
if (!intel_pstate_driver)
@@ -2532,8 +2801,6 @@
static int intel_pstate_update_status(const char *buf, size_t size)
{
- int ret;
-
if (size == 3 && !strncmp(buf, "off", size)) {
if (!intel_pstate_driver)
return -EINVAL;
@@ -2541,7 +2808,9 @@
if (hwp_active)
return -EBUSY;
- return intel_pstate_unregister_driver();
+ cpufreq_unregister_driver(intel_pstate_driver);
+ intel_pstate_driver_cleanup();
+ return 0;
}
if (size == 6 && !strncmp(buf, "active", size)) {
@@ -2549,9 +2818,7 @@
if (intel_pstate_driver == &intel_pstate)
return 0;
- ret = intel_pstate_unregister_driver();
- if (ret)
- return ret;
+ cpufreq_unregister_driver(intel_pstate_driver);
}
return intel_pstate_register_driver(&intel_pstate);
@@ -2562,9 +2829,8 @@
if (intel_pstate_driver == &intel_cpufreq)
return 0;
- ret = intel_pstate_unregister_driver();
- if (ret)
- return ret;
+ cpufreq_unregister_driver(intel_pstate_driver);
+ intel_pstate_sysfs_hide_hwp_dynamic_boost();
}
return intel_pstate_register_driver(&intel_cpufreq);
@@ -2672,24 +2938,26 @@
/* Hardware vendor-specific info that has its own power management modes */
static struct acpi_platform_list plat_info[] __initdata = {
- {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, 0, PSS},
- {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
- {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+ {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS},
+ {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
{ } /* End */
};
+#define BITMASK_OOB (BIT(8) | BIT(18))
+
static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
{
const struct x86_cpu_id *id;
@@ -2699,8 +2967,9 @@
id = x86_match_cpu(intel_pstate_cpu_oob_ids);
if (id) {
rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
- if (misc_pwr & (1 << 8)) {
- pr_debug("Bit 8 in the MISC_PWR_MGMT MSR set\n");
+ if (misc_pwr & BITMASK_OOB) {
+ pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
+ pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
return true;
}
}
@@ -2739,16 +3008,25 @@
#define INTEL_PSTATE_HWP_BROADWELL 0x01
-#define ICPU_HWP(model, hwp_mode) \
- { X86_VENDOR_INTEL, 6, model, X86_FEATURE_HWP, hwp_mode }
+#define X86_MATCH_HWP(model, hwp_mode) \
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
+ X86_FEATURE_HWP, hwp_mode)
static const struct x86_cpu_id hwp_support_ids[] __initconst = {
- ICPU_HWP(INTEL_FAM6_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
- ICPU_HWP(INTEL_FAM6_BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
- ICPU_HWP(X86_MODEL_ANY, 0),
+ X86_MATCH_HWP(BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
+ X86_MATCH_HWP(BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
+ X86_MATCH_HWP(ANY, 0),
{}
};
+static bool intel_pstate_hwp_is_enabled(void)
+{
+ u64 value;
+
+ rdmsrl(MSR_PM_ENABLE, value);
+ return !!(value & 0x1);
+}
+
static int __init intel_pstate_init(void)
{
const struct x86_cpu_id *id;
@@ -2757,19 +3035,40 @@
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
return -ENODEV;
- if (no_load)
- return -ENODEV;
-
id = x86_match_cpu(hwp_support_ids);
if (id) {
+ bool hwp_forced = intel_pstate_hwp_is_enabled();
+
+ if (hwp_forced)
+ pr_info("HWP enabled by BIOS\n");
+ else if (no_load)
+ return -ENODEV;
+
copy_cpu_funcs(&core_funcs);
- if (!no_hwp) {
+ /*
+ * Avoid enabling HWP for processors without EPP support,
+ * because that means incomplete HWP implementation which is a
+ * corner case and supporting it is generally problematic.
+ *
+ * If HWP is enabled already, though, there is no choice but to
+ * deal with it.
+ */
+ if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) || hwp_forced) {
hwp_active++;
hwp_mode_bdw = id->driver_data;
intel_pstate.attr = hwp_cpufreq_attrs;
+ intel_cpufreq.attr = hwp_cpufreq_attrs;
+ intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS;
+ if (!default_driver)
+ default_driver = &intel_pstate;
+
goto hwp_cpu_matched;
}
+ pr_info("HWP not enabled\n");
} else {
+ if (no_load)
+ return -ENODEV;
+
id = x86_match_cpu(intel_pstate_cpu_ids);
if (!id) {
pr_info("CPU model not supported\n");
@@ -2783,6 +3082,9 @@
pr_info("Invalid MSRs\n");
return -ENODEV;
}
+ /* Without HWP start in the passive mode. */
+ if (!default_driver)
+ default_driver = &intel_cpufreq;
hwp_cpu_matched:
/*
@@ -2810,11 +3112,22 @@
mutex_lock(&intel_pstate_driver_lock);
rc = intel_pstate_register_driver(default_driver);
mutex_unlock(&intel_pstate_driver_lock);
- if (rc)
+ if (rc) {
+ intel_pstate_sysfs_remove();
return rc;
+ }
- if (hwp_active)
+ if (hwp_active) {
+ const struct x86_cpu_id *id;
+
+ id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
+ if (id) {
+ set_power_ctl_ee_state(false);
+ pr_info("Disabling energy efficiency optimization\n");
+ }
+
pr_info("HWP enabled\n");
+ }
return 0;
}
@@ -2825,17 +3138,16 @@
if (!str)
return -EINVAL;
- if (!strcmp(str, "disable")) {
+ if (!strcmp(str, "disable"))
no_load = 1;
- } else if (!strcmp(str, "passive")) {
- pr_info("Passive mode enabled\n");
+ else if (!strcmp(str, "active"))
+ default_driver = &intel_pstate;
+ else if (!strcmp(str, "passive"))
default_driver = &intel_cpufreq;
+
+ if (!strcmp(str, "no_hwp"))
no_hwp = 1;
- }
- if (!strcmp(str, "no_hwp")) {
- pr_info("HWP disabled\n");
- no_hwp = 1;
- }
+
if (!strcmp(str, "force"))
force_load = 1;
if (!strcmp(str, "hwp_only"))