[thirdparty/linux.git] / drivers / platform / x86 / intel-uncore-frequency.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Intel Uncore Frequency Setting
 * Copyright (c) 2019, Intel Corporation.
 * All rights reserved.
 *
 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
 * one control CPU is identified per die to read/write limit. This control CPU
 * is changed, if the CPU state is changed to offline. When the last CPU is
 * offline in a die then remove the sysfs object for that die.
 * The majority of actual code is related to sysfs create and read/write
 * attributes.
 *
 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
 */

#include <linux/cpu.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/suspend.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>

#define MSR_UNCORE_RATIO_LIMIT			0x620
#define UNCORE_FREQ_KHZ_MULTIPLIER		100000

/**
 * struct uncore_data -	Encapsulate all uncore data
 * @stored_uncore_data:	Last user changed MSR 620 value, which will be restored
 *			on system resume.
 * @initial_min_freq_khz: Sampled minimum uncore frequency at driver init
 * @initial_max_freq_khz: Sampled maximum uncore frequency at driver init
 * @control_cpu:	Designated CPU for a die to read/write
 * @valid:		Mark the data valid/invalid
 *
 * This structure is used to encapsulate all data related to uncore sysfs
 * settings for a die/package.
 */
struct uncore_data {
	struct kobject kobj;
	struct completion kobj_unregister;
	u64 stored_uncore_data;
	u32 initial_min_freq_khz;
	u32 initial_max_freq_khz;
	int control_cpu;
	bool valid;
};

#define to_uncore_data(a) container_of(a, struct uncore_data, kobj)

/* Max instances for uncore data, one for each die */
static int uncore_max_entries __read_mostly;
/* Storage for uncore data for all instances */
static struct uncore_data *uncore_instances;
/* Root of the all uncore sysfs kobjs */
static struct kobject *uncore_root_kobj;
/* Stores the CPU mask of the target CPUs to use during uncore read/write */
static cpumask_t uncore_cpu_mask;
/* CPU online callback register instance */
static enum cpuhp_state uncore_hp_state __read_mostly;
/* Mutex to control all mutual exclusions */
static DEFINE_MUTEX(uncore_lock);

struct uncore_attr {
	struct attribute attr;
	ssize_t (*show)(struct kobject *kobj,
			struct attribute *attr, char *buf);
	ssize_t (*store)(struct kobject *kobj,
			 struct attribute *attr, const char *c, ssize_t count);
};

#define define_one_uncore_ro(_name) \
static struct uncore_attr _name = \
__ATTR(_name, 0444, show_##_name, NULL)

#define define_one_uncore_rw(_name) \
static struct uncore_attr _name = \
__ATTR(_name, 0644, show_##_name, store_##_name)

#define show_uncore_data(member_name)					\
	static ssize_t show_##member_name(struct kobject *kobj,         \
					  struct attribute *attr,	\
					  char *buf)			\
	{                                                               \
		struct uncore_data *data = to_uncore_data(kobj);	\
		return scnprintf(buf, PAGE_SIZE, "%u\n",		\
				 data->member_name);			\
	}								\
	define_one_uncore_ro(member_name)

show_uncore_data(initial_min_freq_khz);
show_uncore_data(initial_max_freq_khz);

/* Common function to read MSR 0x620 and read min/max */
static int uncore_read_ratio(struct uncore_data *data, unsigned int *min,
			     unsigned int *max)
{
	u64 cap;
	int ret;

	if (data->control_cpu < 0)
		return -ENXIO;

	ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
	if (ret)
		return ret;

	*max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
	*min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;

	return 0;
}

/* Common function to set min/max ratios to be used by sysfs callbacks */
static int uncore_write_ratio(struct uncore_data *data, unsigned int input,
			      int set_max)
{
	int ret;
	u64 cap;

	mutex_lock(&uncore_lock);

	if (data->control_cpu < 0) {
		ret = -ENXIO;
		goto finish_write;
	}

	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
	if (!input || input > 0x7F) {
		ret = -EINVAL;
		goto finish_write;
	}

	ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
	if (ret)
		goto finish_write;

	if (set_max) {
		cap &= ~0x7F;
		cap |= input;
	} else  {
		cap &= ~GENMASK(14, 8);
		cap |= (input << 8);
	}

	ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
	if (ret)
		goto finish_write;

	data->stored_uncore_data = cap;

finish_write:
	mutex_unlock(&uncore_lock);

	return ret;
}

static ssize_t store_min_max_freq_khz(struct kobject *kobj,
				      struct attribute *attr,
				      const char *buf, ssize_t count,
				      int min_max)
{
	struct uncore_data *data = to_uncore_data(kobj);
	unsigned int input;

	if (kstrtouint(buf, 10, &input))
		return -EINVAL;

	uncore_write_ratio(data, input, min_max);

	return count;
}

static ssize_t show_min_max_freq_khz(struct kobject *kobj,
				     struct attribute *attr,
				     char *buf, int min_max)
{
	struct uncore_data *data = to_uncore_data(kobj);
	unsigned int min, max;
	int ret;

	mutex_lock(&uncore_lock);
	ret = uncore_read_ratio(data, &min, &max);
	mutex_unlock(&uncore_lock);
	if (ret)
		return ret;

	if (min_max)
		return sprintf(buf, "%u\n", max);

	return sprintf(buf, "%u\n", min);
}

#define store_uncore_min_max(name, min_max)				\
	static ssize_t store_##name(struct kobject *kobj,		\
				    struct attribute *attr,		\
				    const char *buf, ssize_t count)	\
	{                                                               \
									\
		return store_min_max_freq_khz(kobj, attr, buf, count,	\
					      min_max);			\
	}

#define show_uncore_min_max(name, min_max)				\
	static ssize_t show_##name(struct kobject *kobj,		\
				   struct attribute *attr, char *buf)	\
	{                                                               \
									\
		return show_min_max_freq_khz(kobj, attr, buf, min_max); \
	}

store_uncore_min_max(min_freq_khz, 0);
store_uncore_min_max(max_freq_khz, 1);

show_uncore_min_max(min_freq_khz, 0);
show_uncore_min_max(max_freq_khz, 1);

define_one_uncore_rw(min_freq_khz);
define_one_uncore_rw(max_freq_khz);

static struct attribute *uncore_attrs[] = {
	&initial_min_freq_khz.attr,
	&initial_max_freq_khz.attr,
	&max_freq_khz.attr,
	&min_freq_khz.attr,
	NULL
};

static void uncore_sysfs_entry_release(struct kobject *kobj)
{
	struct uncore_data *data = to_uncore_data(kobj);

	complete(&data->kobj_unregister);
}

static struct kobj_type uncore_ktype = {
	.release = uncore_sysfs_entry_release,
	.sysfs_ops = &kobj_sysfs_ops,
	.default_attrs = uncore_attrs,
};

/* Caller provides protection */
static struct uncore_data *uncore_get_instance(unsigned int cpu)
{
	int id = topology_logical_die_id(cpu);

	if (id >= 0 && id < uncore_max_entries)
		return &uncore_instances[id];

	return NULL;
}

static void uncore_add_die_entry(int cpu)
{
	struct uncore_data *data;

	mutex_lock(&uncore_lock);
	data = uncore_get_instance(cpu);
	if (!data) {
		mutex_unlock(&uncore_lock);
		return;
	}

	if (data->valid) {
		/* control cpu changed */
		data->control_cpu = cpu;
	} else {
		char str[64];
		int ret;

		memset(data, 0, sizeof(*data));
		sprintf(str, "package_%02d_die_%02d",
			topology_physical_package_id(cpu),
			topology_die_id(cpu));

		uncore_read_ratio(data, &data->initial_min_freq_khz,
				  &data->initial_max_freq_khz);

		init_completion(&data->kobj_unregister);

		ret = kobject_init_and_add(&data->kobj, &uncore_ktype,
					   uncore_root_kobj, str);
		if (!ret) {
			data->control_cpu = cpu;
			data->valid = true;
		}
	}
	mutex_unlock(&uncore_lock);
}

/* Last CPU in this die is offline, make control cpu invalid */
static void uncore_remove_die_entry(int cpu)
{
	struct uncore_data *data;

	mutex_lock(&uncore_lock);
	data = uncore_get_instance(cpu);
	if (data)
		data->control_cpu = -1;
	mutex_unlock(&uncore_lock);
}

static int uncore_event_cpu_online(unsigned int cpu)
{
	int target;

	/* Check if there is an online cpu in the package for uncore MSR */
	target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
	if (target < nr_cpu_ids)
		return 0;

	/* Use this CPU on this die as a control CPU */
	cpumask_set_cpu(cpu, &uncore_cpu_mask);
	uncore_add_die_entry(cpu);

	return 0;
}

static int uncore_event_cpu_offline(unsigned int cpu)
{
	int target;

	/* Check if existing cpu is used for uncore MSRs */
	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
		return 0;

	/* Find a new cpu to set uncore MSR */
	target = cpumask_any_but(topology_die_cpumask(cpu), cpu);

	if (target < nr_cpu_ids) {
		cpumask_set_cpu(target, &uncore_cpu_mask);
		uncore_add_die_entry(target);
	} else {
		uncore_remove_die_entry(cpu);
	}

	return 0;
}

static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
			    void *_unused)
{
	int cpu;

	switch (mode) {
	case PM_POST_HIBERNATION:
	case PM_POST_RESTORE:
	case PM_POST_SUSPEND:
		for_each_cpu(cpu, &uncore_cpu_mask) {
			struct uncore_data *data;
			int ret;

			data = uncore_get_instance(cpu);
			if (!data || !data->valid || !data->stored_uncore_data)
				continue;

			ret = wrmsrl_on_cpu(cpu, MSR_UNCORE_RATIO_LIMIT,
					    data->stored_uncore_data);
			if (ret)
				return ret;
		}
		break;
	default:
		break;
	}
	return 0;
}

static struct notifier_block uncore_pm_nb = {
	.notifier_call = uncore_pm_notify,
};

static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,	NULL),
	{}
};

static int __init intel_uncore_init(void)
{
	const struct x86_cpu_id *id;
	int ret;

	id = x86_match_cpu(intel_uncore_cpu_ids);
	if (!id)
		return -ENODEV;

	uncore_max_entries = topology_max_packages() *
					topology_max_die_per_package();
	uncore_instances = kcalloc(uncore_max_entries,
				   sizeof(*uncore_instances), GFP_KERNEL);
	if (!uncore_instances)
		return -ENOMEM;

	uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
						  &cpu_subsys.dev_root->kobj);
	if (!uncore_root_kobj) {
		ret = -ENOMEM;
		goto err_free;
	}

	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
				"platform/x86/uncore-freq:online",
				uncore_event_cpu_online,
				uncore_event_cpu_offline);
	if (ret < 0)
		goto err_rem_kobj;

	uncore_hp_state = ret;

	ret = register_pm_notifier(&uncore_pm_nb);
	if (ret)
		goto err_rem_state;

	return 0;

err_rem_state:
	cpuhp_remove_state(uncore_hp_state);
err_rem_kobj:
	kobject_put(uncore_root_kobj);
err_free:
	kfree(uncore_instances);

	return ret;
}
module_init(intel_uncore_init)

static void __exit intel_uncore_exit(void)
{
	int i;

	unregister_pm_notifier(&uncore_pm_nb);
	cpuhp_remove_state(uncore_hp_state);
	for (i = 0; i < uncore_max_entries; ++i) {
		if (uncore_instances[i].valid) {
			kobject_put(&uncore_instances[i].kobj);
			wait_for_completion(&uncore_instances[i].kobj_unregister);
		}
	}
	kobject_put(uncore_root_kobj);
	kfree(uncore_instances);
}
module_exit(intel_uncore_exit)

MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
Commit	Line	Data
49a474c7 SP	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Intel Uncore Frequency Setting
	4	* Copyright (c) 2019, Intel Corporation.
	5	* All rights reserved.
	6	*
	7	* Provide interface to set MSR 620 at a granularity of per die. On CPU online,
	8	* one control CPU is identified per die to read/write limit. This control CPU
	9	* is changed, if the CPU state is changed to offline. When the last CPU is
	10	* offline in a die then remove the sysfs object for that die.
	11	* The majority of actual code is related to sysfs create and read/write
	12	* attributes.
	13	*
	14	* Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
	15	*/
	16
	17	#include <linux/cpu.h>
	18	#include <linux/module.h>
	19	#include <linux/slab.h>
	20	#include <linux/suspend.h>
	21	#include <asm/cpu_device_id.h>
	22	#include <asm/intel-family.h>
	23
	24	#define MSR_UNCORE_RATIO_LIMIT 0x620
	25	#define UNCORE_FREQ_KHZ_MULTIPLIER 100000
	26
	27	/**
	28	* struct uncore_data - Encapsulate all uncore data
	29	* @stored_uncore_data: Last user changed MSR 620 value, which will be restored
	30	* on system resume.
	31	* @initial_min_freq_khz: Sampled minimum uncore frequency at driver init
	32	* @initial_max_freq_khz: Sampled maximum uncore frequency at driver init
	33	* @control_cpu: Designated CPU for a die to read/write
	34	* @valid: Mark the data valid/invalid
	35	*
	36	* This structure is used to encapsulate all data related to uncore sysfs
	37	* settings for a die/package.
	38	*/
	39	struct uncore_data {
	40	struct kobject kobj;
ee633afd	41	struct completion kobj_unregister;
49a474c7 SP	42	u64 stored_uncore_data;
	43	u32 initial_min_freq_khz;
	44	u32 initial_max_freq_khz;
	45	int control_cpu;
	46	bool valid;
	47	};
	48
	49	#define to_uncore_data(a) container_of(a, struct uncore_data, kobj)
	50
	51	/* Max instances for uncore data, one for each die */
	52	static int uncore_max_entries __read_mostly;
	53	/* Storage for uncore data for all instances */
	54	static struct uncore_data *uncore_instances;
	55	/* Root of the all uncore sysfs kobjs */
f585c9d5	56	static struct kobject *uncore_root_kobj;
49a474c7 SP	57	/* Stores the CPU mask of the target CPUs to use during uncore read/write */
	58	static cpumask_t uncore_cpu_mask;
	59	/* CPU online callback register instance */
	60	static enum cpuhp_state uncore_hp_state __read_mostly;
	61	/* Mutex to control all mutual exclusions */
	62	static DEFINE_MUTEX(uncore_lock);
	63
	64	struct uncore_attr {
	65	struct attribute attr;
	66	ssize_t (show)(struct kobject kobj,
	67	struct attribute attr, char buf);
	68	ssize_t (store)(struct kobject kobj,
	69	struct attribute attr, const char c, ssize_t count);
	70	};
	71
	72	#define define_one_uncore_ro(_name) \
	73	static struct uncore_attr _name = \
	74	__ATTR(_name, 0444, show_##_name, NULL)
	75
	76	#define define_one_uncore_rw(_name) \
	77	static struct uncore_attr _name = \
	78	__ATTR(_name, 0644, show_##_name, store_##_name)
	79
	80	#define show_uncore_data(member_name) \
	81	static ssize_t show_##member_name(struct kobject *kobj, \
	82	struct attribute *attr, \
	83	char *buf) \
	84	{ \
	85	struct uncore_data *data = to_uncore_data(kobj); \
	86	return scnprintf(buf, PAGE_SIZE, "%u\n", \
	87	data->member_name); \
	88	} \
	89	define_one_uncore_ro(member_name)
	90
	91	show_uncore_data(initial_min_freq_khz);
	92	show_uncore_data(initial_max_freq_khz);
	93
	94	/* Common function to read MSR 0x620 and read min/max */
	95	static int uncore_read_ratio(struct uncore_data data, unsigned int min,
	96	unsigned int *max)
	97	{
	98	u64 cap;
	99	int ret;
	100
64b73cff SP	101	if (data->control_cpu < 0)
	102	return -ENXIO;
	103
49a474c7 SP	104	ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
	105	if (ret)
	106	return ret;
	107
	108	max = (cap & 0x7F) UNCORE_FREQ_KHZ_MULTIPLIER;
	109	min = ((cap & GENMASK(14, 8)) >> 8) UNCORE_FREQ_KHZ_MULTIPLIER;
	110
	111	return 0;
	112	}
	113
	114	/* Common function to set min/max ratios to be used by sysfs callbacks */
	115	static int uncore_write_ratio(struct uncore_data *data, unsigned int input,
	116	int set_max)
	117	{
	118	int ret;
	119	u64 cap;
	120
	121	mutex_lock(&uncore_lock);
	122
64b73cff SP	123	if (data->control_cpu < 0) {
	124	ret = -ENXIO;
	125	goto finish_write;
	126	}
	127
49a474c7 SP	128	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
	129	if (!input \|\| input > 0x7F) {
	130	ret = -EINVAL;
	131	goto finish_write;
	132	}
	133
	134	ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
	135	if (ret)
	136	goto finish_write;
	137
	138	if (set_max) {
	139	cap &= ~0x7F;
	140	cap \|= input;
	141	} else {
	142	cap &= ~GENMASK(14, 8);
	143	cap \|= (input << 8);
	144	}
	145
	146	ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
	147	if (ret)
	148	goto finish_write;
	149
	150	data->stored_uncore_data = cap;
	151
	152	finish_write:
	153	mutex_unlock(&uncore_lock);
	154
	155	return ret;
	156	}
	157
	158	static ssize_t store_min_max_freq_khz(struct kobject *kobj,
	159	struct attribute *attr,
	160	const char *buf, ssize_t count,
	161	int min_max)
	162	{
	163	struct uncore_data *data = to_uncore_data(kobj);
	164	unsigned int input;
	165
	166	if (kstrtouint(buf, 10, &input))
	167	return -EINVAL;
	168
	169	uncore_write_ratio(data, input, min_max);
	170
	171	return count;
	172	}
	173
	174	static ssize_t show_min_max_freq_khz(struct kobject *kobj,
	175	struct attribute *attr,
	176	char *buf, int min_max)
	177	{
	178	struct uncore_data *data = to_uncore_data(kobj);
	179	unsigned int min, max;
	180	int ret;
	181
	182	mutex_lock(&uncore_lock);
	183	ret = uncore_read_ratio(data, &min, &max);
	184	mutex_unlock(&uncore_lock);
	185	if (ret)
	186	return ret;
	187
	188	if (min_max)
	189	return sprintf(buf, "%u\n", max);
	190
	191	return sprintf(buf, "%u\n", min);
192	}
193
194	#define store_uncore_min_max(name, min_max) \
195	static ssize_t store_##name(struct kobject *kobj, \
196	struct attribute *attr, \
197	const char *buf, ssize_t count) \
198	{ \
199	\
200	return store_min_max_freq_khz(kobj, attr, buf, count, \
201	min_max); \
202	}
203
204	#define show_uncore_min_max(name, min_max) \
205	static ssize_t show_##name(struct kobject *kobj, \
206	struct attribute attr, char buf) \
207	{ \
208	\
209	return show_min_max_freq_khz(kobj, attr, buf, min_max); \
210	}
211
212	store_uncore_min_max(min_freq_khz, 0);
213	store_uncore_min_max(max_freq_khz, 1);
214
215	show_uncore_min_max(min_freq_khz, 0);
216	show_uncore_min_max(max_freq_khz, 1);
217
218	define_one_uncore_rw(min_freq_khz);
219	define_one_uncore_rw(max_freq_khz);
220
221	static struct attribute *uncore_attrs[] = {
222	&initial_min_freq_khz.attr,
223	&initial_max_freq_khz.attr,
224	&max_freq_khz.attr,
225	&min_freq_khz.attr,
226	NULL
227	};
228
ee633afd SP	229	static void uncore_sysfs_entry_release(struct kobject *kobj)
	230	{
	231	struct uncore_data *data = to_uncore_data(kobj);
	232
	233	complete(&data->kobj_unregister);
	234	}
	235
49a474c7	236	static struct kobj_type uncore_ktype = {
ee633afd	237	.release = uncore_sysfs_entry_release,
49a474c7 SP	238	.sysfs_ops = &kobj_sysfs_ops,
	239	.default_attrs = uncore_attrs,
	240	};
	241
49a474c7 SP	242	/* Caller provides protection */
	243	static struct uncore_data *uncore_get_instance(unsigned int cpu)
	244	{
	245	int id = topology_logical_die_id(cpu);
	246
	247	if (id >= 0 && id < uncore_max_entries)
	248	return &uncore_instances[id];
	249
	250	return NULL;
	251	}
	252
	253	static void uncore_add_die_entry(int cpu)
	254	{
	255	struct uncore_data *data;
	256
	257	mutex_lock(&uncore_lock);
	258	data = uncore_get_instance(cpu);
	259	if (!data) {
	260	mutex_unlock(&uncore_lock);
	261	return;
	262	}
	263
	264	if (data->valid) {
	265	/* control cpu changed */
	266	data->control_cpu = cpu;
	267	} else {
	268	char str[64];
	269	int ret;
	270
	271	memset(data, 0, sizeof(*data));
	272	sprintf(str, "package_%02d_die_%02d",
	273	topology_physical_package_id(cpu),
	274	topology_die_id(cpu));
	275
	276	uncore_read_ratio(data, &data->initial_min_freq_khz,
	277	&data->initial_max_freq_khz);
	278
ee633afd SP	279	init_completion(&data->kobj_unregister);
ee633afd SP	280
49a474c7	281	ret = kobject_init_and_add(&data->kobj, &uncore_ktype,
ee633afd	282	uncore_root_kobj, str);
49a474c7 SP	283	if (!ret) {
	284	data->control_cpu = cpu;
	285	data->valid = true;
	286	}
	287	}
	288	mutex_unlock(&uncore_lock);
	289	}
	290
64b73cff	291	/* Last CPU in this die is offline, make control cpu invalid */
49a474c7 SP	292	static void uncore_remove_die_entry(int cpu)
	293	{
	294	struct uncore_data *data;
	295
	296	mutex_lock(&uncore_lock);
	297	data = uncore_get_instance(cpu);
64b73cff	298	if (data)
49a474c7	299	data->control_cpu = -1;
49a474c7 SP	300	mutex_unlock(&uncore_lock);
	301	}
	302
	303	static int uncore_event_cpu_online(unsigned int cpu)
	304	{
	305	int target;
	306
	307	/* Check if there is an online cpu in the package for uncore MSR */
	308	target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
	309	if (target < nr_cpu_ids)
	310	return 0;
	311
	312	/* Use this CPU on this die as a control CPU */
	313	cpumask_set_cpu(cpu, &uncore_cpu_mask);
	314	uncore_add_die_entry(cpu);
	315
	316	return 0;
	317	}
	318
	319	static int uncore_event_cpu_offline(unsigned int cpu)
	320	{
	321	int target;
	322
	323	/* Check if existing cpu is used for uncore MSRs */
	324	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
	325	return 0;
	326
	327	/* Find a new cpu to set uncore MSR */
	328	target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
	329
	330	if (target < nr_cpu_ids) {
	331	cpumask_set_cpu(target, &uncore_cpu_mask);
	332	uncore_add_die_entry(target);
	333	} else {
	334	uncore_remove_die_entry(cpu);
	335	}
	336
	337	return 0;
	338	}
	339
	340	static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
	341	void *_unused)
	342	{
	343	int cpu;
	344
	345	switch (mode) {
	346	case PM_POST_HIBERNATION:
	347	case PM_POST_RESTORE:
	348	case PM_POST_SUSPEND:
	349	for_each_cpu(cpu, &uncore_cpu_mask) {
	350	struct uncore_data *data;
	351	int ret;
	352
	353	data = uncore_get_instance(cpu);
	354	if (!data \|\| !data->valid \|\| !data->stored_uncore_data)
	355	continue;
	356
	357	ret = wrmsrl_on_cpu(cpu, MSR_UNCORE_RATIO_LIMIT,
	358	data->stored_uncore_data);
	359	if (ret)
	360	return ret;
	361	}
	362	break;
	363	default:
364	break;
365	}
366	return 0;
367	}
368
369	static struct notifier_block uncore_pm_nb = {
370	.notifier_call = uncore_pm_notify,
371	};
372
49a474c7	373	static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
a69b3b1d TG	374	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, NULL),
	375	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, NULL),
	376	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, NULL),
	377	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
	378	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
	379	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
49a474c7 SP	380	{}
	381	};
	382
	383	static int __init intel_uncore_init(void)
	384	{
	385	const struct x86_cpu_id *id;
	386	int ret;
	387
	388	id = x86_match_cpu(intel_uncore_cpu_ids);
	389	if (!id)
	390	return -ENODEV;
	391
	392	uncore_max_entries = topology_max_packages() *
	393	topology_max_die_per_package();
	394	uncore_instances = kcalloc(uncore_max_entries,
	395	sizeof(*uncore_instances), GFP_KERNEL);
	396	if (!uncore_instances)
	397	return -ENOMEM;
	398
ee633afd SP	399	uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
	400	&cpu_subsys.dev_root->kobj);
	401	if (!uncore_root_kobj) {
	402	ret = -ENOMEM;
49a474c7	403	goto err_free;
ee633afd	404	}
49a474c7 SP	405
	406	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
	407	"platform/x86/uncore-freq:online",
	408	uncore_event_cpu_online,
	409	uncore_event_cpu_offline);
	410	if (ret < 0)
	411	goto err_rem_kobj;
	412
	413	uncore_hp_state = ret;
	414
	415	ret = register_pm_notifier(&uncore_pm_nb);
	416	if (ret)
	417	goto err_rem_state;
	418
	419	return 0;
	420
	421	err_rem_state:
	422	cpuhp_remove_state(uncore_hp_state);
	423	err_rem_kobj:
ee633afd	424	kobject_put(uncore_root_kobj);
49a474c7 SP	425	err_free:
	426	kfree(uncore_instances);
	427
	428	return ret;
	429	}
	430	module_init(intel_uncore_init)
	431
	432	static void __exit intel_uncore_exit(void)
	433	{
	434	int i;
	435
	436	unregister_pm_notifier(&uncore_pm_nb);
	437	cpuhp_remove_state(uncore_hp_state);
	438	for (i = 0; i < uncore_max_entries; ++i) {
ee633afd	439	if (uncore_instances[i].valid) {
49a474c7	440	kobject_put(&uncore_instances[i].kobj);
ee633afd SP	441	wait_for_completion(&uncore_instances[i].kobj_unregister);
ee633afd SP	442	}
49a474c7	443	}
ee633afd	444	kobject_put(uncore_root_kobj);
49a474c7 SP	445	kfree(uncore_instances);
	446	}
	447	module_exit(intel_uncore_exit)
	448
	449	MODULE_LICENSE("GPL v2");
	450	MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");