1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * amd-pstate.c - AMD Processor P-state Frequency Driver
5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
7 * Author: Huang Rui <ray.huang@amd.com>
9 * AMD P-State introduces a new CPU performance scaling design for AMD
10 * processors using the ACPI Collaborative Performance and Power Control (CPPC)
11 * feature which works with the AMD SMU firmware providing a finer grained
12 * frequency control range. It is to replace the legacy ACPI P-States control,
13 * allows a flexible, low-latency interface for the Linux kernel to directly
14 * communicate the performance hints to hardware.
16 * AMD P-State is supported on recent AMD Zen base CPU series include some of
17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
18 * P-State supported system. And there are two types of hardware implementations
19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25 #include <linux/kernel.h>
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/smp.h>
29 #include <linux/sched.h>
30 #include <linux/cpufreq.h>
31 #include <linux/compiler.h>
32 #include <linux/dmi.h>
33 #include <linux/slab.h>
34 #include <linux/acpi.h>
36 #include <linux/delay.h>
37 #include <linux/uaccess.h>
38 #include <linux/static_call.h>
39 #include <linux/amd-pstate.h>
41 #include <acpi/processor.h>
42 #include <acpi/cppc_acpi.h>
45 #include <asm/processor.h>
46 #include <asm/cpufeature.h>
47 #include <asm/cpu_device_id.h>
48 #include "amd-pstate-trace.h"
50 #define AMD_PSTATE_TRANSITION_LATENCY 20000
51 #define AMD_PSTATE_TRANSITION_DELAY 1000
54 * TODO: We need more time to fine tune processors with shared memory solution
55 * with community together.
57 * There are some performance drops on the CPU benchmarks which reports from
58 * Suse. We are co-working with them to fine tune the shared memory solution. So
59 * we disable it by default to go acpi-cpufreq on these processors and add a
60 * module parameter to be able to enable it manually for debugging.
62 static struct cpufreq_driver
*current_pstate_driver
;
63 static struct cpufreq_driver amd_pstate_driver
;
64 static struct cpufreq_driver amd_pstate_epp_driver
;
65 static int cppc_state
= AMD_PSTATE_UNDEFINED
;
66 static bool cppc_enabled
;
69 * AMD Energy Preference Performance (EPP)
70 * The EPP is used in the CCLK DPM controller to drive
71 * the frequency that a core is going to operate during
72 * short periods of activity. EPP values will be utilized for
73 * different OS profiles (balanced, performance, power savings)
74 * display strings corresponding to EPP index in the
75 * energy_perf_strings[]
77 *-------------------------------------
80 * 2 balance_performance
84 enum energy_perf_value_index
{
85 EPP_INDEX_DEFAULT
= 0,
86 EPP_INDEX_PERFORMANCE
,
87 EPP_INDEX_BALANCE_PERFORMANCE
,
88 EPP_INDEX_BALANCE_POWERSAVE
,
92 static const char * const energy_perf_strings
[] = {
93 [EPP_INDEX_DEFAULT
] = "default",
94 [EPP_INDEX_PERFORMANCE
] = "performance",
95 [EPP_INDEX_BALANCE_PERFORMANCE
] = "balance_performance",
96 [EPP_INDEX_BALANCE_POWERSAVE
] = "balance_power",
97 [EPP_INDEX_POWERSAVE
] = "power",
101 static unsigned int epp_values
[] = {
102 [EPP_INDEX_DEFAULT
] = 0,
103 [EPP_INDEX_PERFORMANCE
] = AMD_CPPC_EPP_PERFORMANCE
,
104 [EPP_INDEX_BALANCE_PERFORMANCE
] = AMD_CPPC_EPP_BALANCE_PERFORMANCE
,
105 [EPP_INDEX_BALANCE_POWERSAVE
] = AMD_CPPC_EPP_BALANCE_POWERSAVE
,
106 [EPP_INDEX_POWERSAVE
] = AMD_CPPC_EPP_POWERSAVE
,
109 typedef int (*cppc_mode_transition_fn
)(int);
111 static inline int get_mode_idx_from_str(const char *str
, size_t size
)
115 for (i
=0; i
< AMD_PSTATE_MAX
; i
++) {
116 if (!strncmp(str
, amd_pstate_mode_string
[i
], size
))
122 static DEFINE_MUTEX(amd_pstate_limits_lock
);
123 static DEFINE_MUTEX(amd_pstate_driver_lock
);
125 static s16
amd_pstate_get_epp(struct amd_cpudata
*cpudata
, u64 cppc_req_cached
)
130 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
131 if (!cppc_req_cached
) {
132 epp
= rdmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_REQ
,
137 epp
= (cppc_req_cached
>> 24) & 0xFF;
139 ret
= cppc_get_epp_perf(cpudata
->cpu
, &epp
);
141 pr_debug("Could not retrieve energy perf value (%d)\n", ret
);
146 return (s16
)(epp
& 0xff);
149 static int amd_pstate_get_energy_pref_index(struct amd_cpudata
*cpudata
)
154 epp
= amd_pstate_get_epp(cpudata
, 0);
159 case AMD_CPPC_EPP_PERFORMANCE
:
160 index
= EPP_INDEX_PERFORMANCE
;
162 case AMD_CPPC_EPP_BALANCE_PERFORMANCE
:
163 index
= EPP_INDEX_BALANCE_PERFORMANCE
;
165 case AMD_CPPC_EPP_BALANCE_POWERSAVE
:
166 index
= EPP_INDEX_BALANCE_POWERSAVE
;
168 case AMD_CPPC_EPP_POWERSAVE
:
169 index
= EPP_INDEX_POWERSAVE
;
178 static int amd_pstate_set_epp(struct amd_cpudata
*cpudata
, u32 epp
)
181 struct cppc_perf_ctrls perf_ctrls
;
183 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
184 u64 value
= READ_ONCE(cpudata
->cppc_req_cached
);
186 value
&= ~GENMASK_ULL(31, 24);
187 value
|= (u64
)epp
<< 24;
188 WRITE_ONCE(cpudata
->cppc_req_cached
, value
);
190 ret
= wrmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_REQ
, value
);
192 cpudata
->epp_cached
= epp
;
194 perf_ctrls
.energy_perf
= epp
;
195 ret
= cppc_set_epp_perf(cpudata
->cpu
, &perf_ctrls
, 1);
197 pr_debug("failed to set energy perf value (%d)\n", ret
);
200 cpudata
->epp_cached
= epp
;
206 static int amd_pstate_set_energy_pref_index(struct amd_cpudata
*cpudata
,
213 pr_debug("EPP pref_index is invalid\n");
218 epp
= epp_values
[pref_index
];
220 if (epp
> 0 && cpudata
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
221 pr_debug("EPP cannot be set under performance policy\n");
225 ret
= amd_pstate_set_epp(cpudata
, epp
);
230 static inline int pstate_enable(bool enable
)
233 unsigned long logical_proc_id_mask
= 0;
235 if (enable
== cppc_enabled
)
238 for_each_present_cpu(cpu
) {
239 unsigned long logical_id
= topology_logical_die_id(cpu
);
241 if (test_bit(logical_id
, &logical_proc_id_mask
))
244 set_bit(logical_id
, &logical_proc_id_mask
);
246 ret
= wrmsrl_safe_on_cpu(cpu
, MSR_AMD_CPPC_ENABLE
,
252 cppc_enabled
= enable
;
256 static int cppc_enable(bool enable
)
259 struct cppc_perf_ctrls perf_ctrls
;
261 if (enable
== cppc_enabled
)
264 for_each_present_cpu(cpu
) {
265 ret
= cppc_set_enable(cpu
, enable
);
269 /* Enable autonomous mode for EPP */
270 if (cppc_state
== AMD_PSTATE_ACTIVE
) {
271 /* Set desired perf as zero to allow EPP firmware control */
272 perf_ctrls
.desired_perf
= 0;
273 ret
= cppc_set_perf(cpu
, &perf_ctrls
);
279 cppc_enabled
= enable
;
283 DEFINE_STATIC_CALL(amd_pstate_enable
, pstate_enable
);
285 static inline int amd_pstate_enable(bool enable
)
287 return static_call(amd_pstate_enable
)(enable
);
290 static int pstate_init_perf(struct amd_cpudata
*cpudata
)
295 int ret
= rdmsrl_safe_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_CAP1
,
301 * TODO: Introduce AMD specific power feature.
303 * CPPC entry doesn't indicate the highest performance in some ASICs.
305 highest_perf
= amd_get_highest_perf();
306 if (highest_perf
> AMD_CPPC_HIGHEST_PERF(cap1
))
307 highest_perf
= AMD_CPPC_HIGHEST_PERF(cap1
);
309 WRITE_ONCE(cpudata
->highest_perf
, highest_perf
);
311 WRITE_ONCE(cpudata
->nominal_perf
, AMD_CPPC_NOMINAL_PERF(cap1
));
312 WRITE_ONCE(cpudata
->lowest_nonlinear_perf
, AMD_CPPC_LOWNONLIN_PERF(cap1
));
313 WRITE_ONCE(cpudata
->lowest_perf
, AMD_CPPC_LOWEST_PERF(cap1
));
318 static int cppc_init_perf(struct amd_cpudata
*cpudata
)
320 struct cppc_perf_caps cppc_perf
;
323 int ret
= cppc_get_perf_caps(cpudata
->cpu
, &cppc_perf
);
327 highest_perf
= amd_get_highest_perf();
328 if (highest_perf
> cppc_perf
.highest_perf
)
329 highest_perf
= cppc_perf
.highest_perf
;
331 WRITE_ONCE(cpudata
->highest_perf
, highest_perf
);
333 WRITE_ONCE(cpudata
->nominal_perf
, cppc_perf
.nominal_perf
);
334 WRITE_ONCE(cpudata
->lowest_nonlinear_perf
,
335 cppc_perf
.lowest_nonlinear_perf
);
336 WRITE_ONCE(cpudata
->lowest_perf
, cppc_perf
.lowest_perf
);
338 if (cppc_state
== AMD_PSTATE_ACTIVE
)
341 ret
= cppc_get_auto_sel_caps(cpudata
->cpu
, &cppc_perf
);
343 pr_warn("failed to get auto_sel, ret: %d\n", ret
);
347 ret
= cppc_set_auto_sel(cpudata
->cpu
,
348 (cppc_state
== AMD_PSTATE_PASSIVE
) ? 0 : 1);
351 pr_warn("failed to set auto_sel, ret: %d\n", ret
);
356 DEFINE_STATIC_CALL(amd_pstate_init_perf
, pstate_init_perf
);
358 static inline int amd_pstate_init_perf(struct amd_cpudata
*cpudata
)
360 return static_call(amd_pstate_init_perf
)(cpudata
);
363 static void pstate_update_perf(struct amd_cpudata
*cpudata
, u32 min_perf
,
364 u32 des_perf
, u32 max_perf
, bool fast_switch
)
367 wrmsrl(MSR_AMD_CPPC_REQ
, READ_ONCE(cpudata
->cppc_req_cached
));
369 wrmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_REQ
,
370 READ_ONCE(cpudata
->cppc_req_cached
));
373 static void cppc_update_perf(struct amd_cpudata
*cpudata
,
374 u32 min_perf
, u32 des_perf
,
375 u32 max_perf
, bool fast_switch
)
377 struct cppc_perf_ctrls perf_ctrls
;
379 perf_ctrls
.max_perf
= max_perf
;
380 perf_ctrls
.min_perf
= min_perf
;
381 perf_ctrls
.desired_perf
= des_perf
;
383 cppc_set_perf(cpudata
->cpu
, &perf_ctrls
);
386 DEFINE_STATIC_CALL(amd_pstate_update_perf
, pstate_update_perf
);
388 static inline void amd_pstate_update_perf(struct amd_cpudata
*cpudata
,
389 u32 min_perf
, u32 des_perf
,
390 u32 max_perf
, bool fast_switch
)
392 static_call(amd_pstate_update_perf
)(cpudata
, min_perf
, des_perf
,
393 max_perf
, fast_switch
);
396 static inline bool amd_pstate_sample(struct amd_cpudata
*cpudata
)
398 u64 aperf
, mperf
, tsc
;
401 local_irq_save(flags
);
402 rdmsrl(MSR_IA32_APERF
, aperf
);
403 rdmsrl(MSR_IA32_MPERF
, mperf
);
406 if (cpudata
->prev
.mperf
== mperf
|| cpudata
->prev
.tsc
== tsc
) {
407 local_irq_restore(flags
);
411 local_irq_restore(flags
);
413 cpudata
->cur
.aperf
= aperf
;
414 cpudata
->cur
.mperf
= mperf
;
415 cpudata
->cur
.tsc
= tsc
;
416 cpudata
->cur
.aperf
-= cpudata
->prev
.aperf
;
417 cpudata
->cur
.mperf
-= cpudata
->prev
.mperf
;
418 cpudata
->cur
.tsc
-= cpudata
->prev
.tsc
;
420 cpudata
->prev
.aperf
= aperf
;
421 cpudata
->prev
.mperf
= mperf
;
422 cpudata
->prev
.tsc
= tsc
;
424 cpudata
->freq
= div64_u64((cpudata
->cur
.aperf
* cpu_khz
), cpudata
->cur
.mperf
);
429 static void amd_pstate_update(struct amd_cpudata
*cpudata
, u32 min_perf
,
430 u32 des_perf
, u32 max_perf
, bool fast_switch
, int gov_flags
)
432 u64 prev
= READ_ONCE(cpudata
->cppc_req_cached
);
435 des_perf
= clamp_t(unsigned long, des_perf
, min_perf
, max_perf
);
437 if ((cppc_state
== AMD_PSTATE_GUIDED
) && (gov_flags
& CPUFREQ_GOV_DYNAMIC_SWITCHING
)) {
442 value
&= ~AMD_CPPC_MIN_PERF(~0L);
443 value
|= AMD_CPPC_MIN_PERF(min_perf
);
445 value
&= ~AMD_CPPC_DES_PERF(~0L);
446 value
|= AMD_CPPC_DES_PERF(des_perf
);
448 value
&= ~AMD_CPPC_MAX_PERF(~0L);
449 value
|= AMD_CPPC_MAX_PERF(max_perf
);
451 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata
)) {
452 trace_amd_pstate_perf(min_perf
, des_perf
, max_perf
, cpudata
->freq
,
453 cpudata
->cur
.mperf
, cpudata
->cur
.aperf
, cpudata
->cur
.tsc
,
454 cpudata
->cpu
, (value
!= prev
), fast_switch
);
460 WRITE_ONCE(cpudata
->cppc_req_cached
, value
);
462 amd_pstate_update_perf(cpudata
, min_perf
, des_perf
,
463 max_perf
, fast_switch
);
466 static int amd_pstate_verify(struct cpufreq_policy_data
*policy
)
468 cpufreq_verify_within_cpu_limits(policy
);
473 static int amd_pstate_update_freq(struct cpufreq_policy
*policy
,
474 unsigned int target_freq
, bool fast_switch
)
476 struct cpufreq_freqs freqs
;
477 struct amd_cpudata
*cpudata
= policy
->driver_data
;
478 unsigned long max_perf
, min_perf
, des_perf
, cap_perf
;
480 if (!cpudata
->max_freq
)
483 cap_perf
= READ_ONCE(cpudata
->highest_perf
);
484 min_perf
= READ_ONCE(cpudata
->lowest_perf
);
487 freqs
.old
= policy
->cur
;
488 freqs
.new = target_freq
;
490 des_perf
= DIV_ROUND_CLOSEST(target_freq
* cap_perf
,
493 WARN_ON(fast_switch
&& !policy
->fast_switch_enabled
);
495 * If fast_switch is desired, then there aren't any registered
496 * transition notifiers. See comment for
497 * cpufreq_enable_fast_switch().
500 cpufreq_freq_transition_begin(policy
, &freqs
);
502 amd_pstate_update(cpudata
, min_perf
, des_perf
,
503 max_perf
, fast_switch
, policy
->governor
->flags
);
506 cpufreq_freq_transition_end(policy
, &freqs
, false);
511 static int amd_pstate_target(struct cpufreq_policy
*policy
,
512 unsigned int target_freq
,
513 unsigned int relation
)
515 return amd_pstate_update_freq(policy
, target_freq
, false);
518 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy
*policy
,
519 unsigned int target_freq
)
521 return amd_pstate_update_freq(policy
, target_freq
, true);
524 static void amd_pstate_adjust_perf(unsigned int cpu
,
525 unsigned long _min_perf
,
526 unsigned long target_perf
,
527 unsigned long capacity
)
529 unsigned long max_perf
, min_perf
, des_perf
,
530 cap_perf
, lowest_nonlinear_perf
, max_freq
;
531 struct cpufreq_policy
*policy
= cpufreq_cpu_get(cpu
);
532 struct amd_cpudata
*cpudata
= policy
->driver_data
;
533 unsigned int target_freq
;
535 cap_perf
= READ_ONCE(cpudata
->highest_perf
);
536 lowest_nonlinear_perf
= READ_ONCE(cpudata
->lowest_nonlinear_perf
);
537 max_freq
= READ_ONCE(cpudata
->max_freq
);
540 if (target_perf
< capacity
)
541 des_perf
= DIV_ROUND_UP(cap_perf
* target_perf
, capacity
);
543 min_perf
= READ_ONCE(cpudata
->highest_perf
);
544 if (_min_perf
< capacity
)
545 min_perf
= DIV_ROUND_UP(cap_perf
* _min_perf
, capacity
);
547 if (min_perf
< lowest_nonlinear_perf
)
548 min_perf
= lowest_nonlinear_perf
;
551 if (max_perf
< min_perf
)
554 des_perf
= clamp_t(unsigned long, des_perf
, min_perf
, max_perf
);
555 target_freq
= div_u64(des_perf
* max_freq
, max_perf
);
556 policy
->cur
= target_freq
;
558 amd_pstate_update(cpudata
, min_perf
, des_perf
, max_perf
, true,
559 policy
->governor
->flags
);
560 cpufreq_cpu_put(policy
);
563 static int amd_get_min_freq(struct amd_cpudata
*cpudata
)
565 struct cppc_perf_caps cppc_perf
;
567 int ret
= cppc_get_perf_caps(cpudata
->cpu
, &cppc_perf
);
572 return cppc_perf
.lowest_freq
* 1000;
575 static int amd_get_max_freq(struct amd_cpudata
*cpudata
)
577 struct cppc_perf_caps cppc_perf
;
578 u32 max_perf
, max_freq
, nominal_freq
, nominal_perf
;
581 int ret
= cppc_get_perf_caps(cpudata
->cpu
, &cppc_perf
);
585 nominal_freq
= cppc_perf
.nominal_freq
;
586 nominal_perf
= READ_ONCE(cpudata
->nominal_perf
);
587 max_perf
= READ_ONCE(cpudata
->highest_perf
);
589 boost_ratio
= div_u64(max_perf
<< SCHED_CAPACITY_SHIFT
,
592 max_freq
= nominal_freq
* boost_ratio
>> SCHED_CAPACITY_SHIFT
;
595 return max_freq
* 1000;
598 static int amd_get_nominal_freq(struct amd_cpudata
*cpudata
)
600 struct cppc_perf_caps cppc_perf
;
602 int ret
= cppc_get_perf_caps(cpudata
->cpu
, &cppc_perf
);
607 return cppc_perf
.nominal_freq
* 1000;
610 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata
*cpudata
)
612 struct cppc_perf_caps cppc_perf
;
613 u32 lowest_nonlinear_freq
, lowest_nonlinear_perf
,
614 nominal_freq
, nominal_perf
;
615 u64 lowest_nonlinear_ratio
;
617 int ret
= cppc_get_perf_caps(cpudata
->cpu
, &cppc_perf
);
621 nominal_freq
= cppc_perf
.nominal_freq
;
622 nominal_perf
= READ_ONCE(cpudata
->nominal_perf
);
624 lowest_nonlinear_perf
= cppc_perf
.lowest_nonlinear_perf
;
626 lowest_nonlinear_ratio
= div_u64(lowest_nonlinear_perf
<< SCHED_CAPACITY_SHIFT
,
629 lowest_nonlinear_freq
= nominal_freq
* lowest_nonlinear_ratio
>> SCHED_CAPACITY_SHIFT
;
632 return lowest_nonlinear_freq
* 1000;
635 static int amd_pstate_set_boost(struct cpufreq_policy
*policy
, int state
)
637 struct amd_cpudata
*cpudata
= policy
->driver_data
;
640 if (!cpudata
->boost_supported
) {
641 pr_err("Boost mode is not supported by this processor or SBIOS\n");
646 policy
->cpuinfo
.max_freq
= cpudata
->max_freq
;
648 policy
->cpuinfo
.max_freq
= cpudata
->nominal_freq
;
650 policy
->max
= policy
->cpuinfo
.max_freq
;
652 ret
= freq_qos_update_request(&cpudata
->req
[1],
653 policy
->cpuinfo
.max_freq
);
660 static void amd_pstate_boost_init(struct amd_cpudata
*cpudata
)
662 u32 highest_perf
, nominal_perf
;
664 highest_perf
= READ_ONCE(cpudata
->highest_perf
);
665 nominal_perf
= READ_ONCE(cpudata
->nominal_perf
);
667 if (highest_perf
<= nominal_perf
)
670 cpudata
->boost_supported
= true;
671 current_pstate_driver
->boost_enabled
= true;
674 static void amd_perf_ctl_reset(unsigned int cpu
)
676 wrmsrl_on_cpu(cpu
, MSR_AMD_PERF_CTL
, 0);
679 static int amd_pstate_cpu_init(struct cpufreq_policy
*policy
)
681 int min_freq
, max_freq
, nominal_freq
, lowest_nonlinear_freq
, ret
;
683 struct amd_cpudata
*cpudata
;
686 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
687 * which is ideal for initialization process.
689 amd_perf_ctl_reset(policy
->cpu
);
690 dev
= get_cpu_device(policy
->cpu
);
694 cpudata
= kzalloc(sizeof(*cpudata
), GFP_KERNEL
);
698 cpudata
->cpu
= policy
->cpu
;
700 ret
= amd_pstate_init_perf(cpudata
);
704 min_freq
= amd_get_min_freq(cpudata
);
705 max_freq
= amd_get_max_freq(cpudata
);
706 nominal_freq
= amd_get_nominal_freq(cpudata
);
707 lowest_nonlinear_freq
= amd_get_lowest_nonlinear_freq(cpudata
);
709 if (min_freq
< 0 || max_freq
< 0 || min_freq
> max_freq
) {
710 dev_err(dev
, "min_freq(%d) or max_freq(%d) value is incorrect\n",
716 policy
->cpuinfo
.transition_latency
= AMD_PSTATE_TRANSITION_LATENCY
;
717 policy
->transition_delay_us
= AMD_PSTATE_TRANSITION_DELAY
;
719 policy
->min
= min_freq
;
720 policy
->max
= max_freq
;
722 policy
->cpuinfo
.min_freq
= min_freq
;
723 policy
->cpuinfo
.max_freq
= max_freq
;
725 /* It will be updated by governor */
726 policy
->cur
= policy
->cpuinfo
.min_freq
;
728 if (boot_cpu_has(X86_FEATURE_CPPC
))
729 policy
->fast_switch_possible
= true;
731 ret
= freq_qos_add_request(&policy
->constraints
, &cpudata
->req
[0],
732 FREQ_QOS_MIN
, policy
->cpuinfo
.min_freq
);
734 dev_err(dev
, "Failed to add min-freq constraint (%d)\n", ret
);
738 ret
= freq_qos_add_request(&policy
->constraints
, &cpudata
->req
[1],
739 FREQ_QOS_MAX
, policy
->cpuinfo
.max_freq
);
741 dev_err(dev
, "Failed to add max-freq constraint (%d)\n", ret
);
745 /* Initial processor data capability frequencies */
746 cpudata
->max_freq
= max_freq
;
747 cpudata
->min_freq
= min_freq
;
748 cpudata
->nominal_freq
= nominal_freq
;
749 cpudata
->lowest_nonlinear_freq
= lowest_nonlinear_freq
;
751 policy
->driver_data
= cpudata
;
753 amd_pstate_boost_init(cpudata
);
754 if (!current_pstate_driver
->adjust_perf
)
755 current_pstate_driver
->adjust_perf
= amd_pstate_adjust_perf
;
760 freq_qos_remove_request(&cpudata
->req
[0]);
766 static int amd_pstate_cpu_exit(struct cpufreq_policy
*policy
)
768 struct amd_cpudata
*cpudata
= policy
->driver_data
;
770 freq_qos_remove_request(&cpudata
->req
[1]);
771 freq_qos_remove_request(&cpudata
->req
[0]);
772 policy
->fast_switch_possible
= false;
778 static int amd_pstate_cpu_resume(struct cpufreq_policy
*policy
)
782 ret
= amd_pstate_enable(true);
784 pr_err("failed to enable amd-pstate during resume, return %d\n", ret
);
789 static int amd_pstate_cpu_suspend(struct cpufreq_policy
*policy
)
793 ret
= amd_pstate_enable(false);
795 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret
);
800 /* Sysfs attributes */
803 * This frequency is to indicate the maximum hardware frequency.
804 * If boost is not active but supported, the frequency will be larger than the
807 static ssize_t
show_amd_pstate_max_freq(struct cpufreq_policy
*policy
,
811 struct amd_cpudata
*cpudata
= policy
->driver_data
;
813 max_freq
= amd_get_max_freq(cpudata
);
817 return sysfs_emit(buf
, "%u\n", max_freq
);
820 static ssize_t
show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy
*policy
,
824 struct amd_cpudata
*cpudata
= policy
->driver_data
;
826 freq
= amd_get_lowest_nonlinear_freq(cpudata
);
830 return sysfs_emit(buf
, "%u\n", freq
);
834 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
835 * need to expose it to sysfs.
837 static ssize_t
show_amd_pstate_highest_perf(struct cpufreq_policy
*policy
,
841 struct amd_cpudata
*cpudata
= policy
->driver_data
;
843 perf
= READ_ONCE(cpudata
->highest_perf
);
845 return sysfs_emit(buf
, "%u\n", perf
);
848 static ssize_t
show_energy_performance_available_preferences(
849 struct cpufreq_policy
*policy
, char *buf
)
854 while (energy_perf_strings
[i
] != NULL
)
855 offset
+= sysfs_emit_at(buf
, offset
, "%s ", energy_perf_strings
[i
++]);
857 sysfs_emit_at(buf
, offset
, "\n");
862 static ssize_t
store_energy_performance_preference(
863 struct cpufreq_policy
*policy
, const char *buf
, size_t count
)
865 struct amd_cpudata
*cpudata
= policy
->driver_data
;
866 char str_preference
[21];
869 ret
= sscanf(buf
, "%20s", str_preference
);
873 ret
= match_string(energy_perf_strings
, -1, str_preference
);
877 mutex_lock(&amd_pstate_limits_lock
);
878 ret
= amd_pstate_set_energy_pref_index(cpudata
, ret
);
879 mutex_unlock(&amd_pstate_limits_lock
);
884 static ssize_t
show_energy_performance_preference(
885 struct cpufreq_policy
*policy
, char *buf
)
887 struct amd_cpudata
*cpudata
= policy
->driver_data
;
890 preference
= amd_pstate_get_energy_pref_index(cpudata
);
894 return sysfs_emit(buf
, "%s\n", energy_perf_strings
[preference
]);
897 static void amd_pstate_driver_cleanup(void)
899 amd_pstate_enable(false);
900 cppc_state
= AMD_PSTATE_DISABLE
;
901 current_pstate_driver
= NULL
;
904 static int amd_pstate_register_driver(int mode
)
908 if (mode
== AMD_PSTATE_PASSIVE
|| mode
== AMD_PSTATE_GUIDED
)
909 current_pstate_driver
= &amd_pstate_driver
;
910 else if (mode
== AMD_PSTATE_ACTIVE
)
911 current_pstate_driver
= &amd_pstate_epp_driver
;
916 ret
= cpufreq_register_driver(current_pstate_driver
);
918 amd_pstate_driver_cleanup();
924 static int amd_pstate_unregister_driver(int dummy
)
926 cpufreq_unregister_driver(current_pstate_driver
);
927 amd_pstate_driver_cleanup();
931 static int amd_pstate_change_mode_without_dvr_change(int mode
)
937 if (boot_cpu_has(X86_FEATURE_CPPC
) || cppc_state
== AMD_PSTATE_ACTIVE
)
940 for_each_present_cpu(cpu
) {
941 cppc_set_auto_sel(cpu
, (cppc_state
== AMD_PSTATE_PASSIVE
) ? 0 : 1);
947 static int amd_pstate_change_driver_mode(int mode
)
951 ret
= amd_pstate_unregister_driver(0);
955 ret
= amd_pstate_register_driver(mode
);
962 static cppc_mode_transition_fn mode_state_machine
[AMD_PSTATE_MAX
][AMD_PSTATE_MAX
] = {
963 [AMD_PSTATE_DISABLE
] = {
964 [AMD_PSTATE_DISABLE
] = NULL
,
965 [AMD_PSTATE_PASSIVE
] = amd_pstate_register_driver
,
966 [AMD_PSTATE_ACTIVE
] = amd_pstate_register_driver
,
967 [AMD_PSTATE_GUIDED
] = amd_pstate_register_driver
,
969 [AMD_PSTATE_PASSIVE
] = {
970 [AMD_PSTATE_DISABLE
] = amd_pstate_unregister_driver
,
971 [AMD_PSTATE_PASSIVE
] = NULL
,
972 [AMD_PSTATE_ACTIVE
] = amd_pstate_change_driver_mode
,
973 [AMD_PSTATE_GUIDED
] = amd_pstate_change_mode_without_dvr_change
,
975 [AMD_PSTATE_ACTIVE
] = {
976 [AMD_PSTATE_DISABLE
] = amd_pstate_unregister_driver
,
977 [AMD_PSTATE_PASSIVE
] = amd_pstate_change_driver_mode
,
978 [AMD_PSTATE_ACTIVE
] = NULL
,
979 [AMD_PSTATE_GUIDED
] = amd_pstate_change_driver_mode
,
981 [AMD_PSTATE_GUIDED
] = {
982 [AMD_PSTATE_DISABLE
] = amd_pstate_unregister_driver
,
983 [AMD_PSTATE_PASSIVE
] = amd_pstate_change_mode_without_dvr_change
,
984 [AMD_PSTATE_ACTIVE
] = amd_pstate_change_driver_mode
,
985 [AMD_PSTATE_GUIDED
] = NULL
,
989 static ssize_t
amd_pstate_show_status(char *buf
)
991 if (!current_pstate_driver
)
992 return sysfs_emit(buf
, "disable\n");
994 return sysfs_emit(buf
, "%s\n", amd_pstate_mode_string
[cppc_state
]);
997 static int amd_pstate_update_status(const char *buf
, size_t size
)
1001 if (size
> strlen("passive") || size
< strlen("active"))
1004 mode_idx
= get_mode_idx_from_str(buf
, size
);
1006 if (mode_idx
< 0 || mode_idx
>= AMD_PSTATE_MAX
)
1009 if (mode_state_machine
[cppc_state
][mode_idx
])
1010 return mode_state_machine
[cppc_state
][mode_idx
](mode_idx
);
1015 static ssize_t
status_show(struct device
*dev
,
1016 struct device_attribute
*attr
, char *buf
)
1020 mutex_lock(&amd_pstate_driver_lock
);
1021 ret
= amd_pstate_show_status(buf
);
1022 mutex_unlock(&amd_pstate_driver_lock
);
1027 static ssize_t
status_store(struct device
*a
, struct device_attribute
*b
,
1028 const char *buf
, size_t count
)
1030 char *p
= memchr(buf
, '\n', count
);
1033 mutex_lock(&amd_pstate_driver_lock
);
1034 ret
= amd_pstate_update_status(buf
, p
? p
- buf
: count
);
1035 mutex_unlock(&amd_pstate_driver_lock
);
1037 return ret
< 0 ? ret
: count
;
1040 cpufreq_freq_attr_ro(amd_pstate_max_freq
);
1041 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq
);
1043 cpufreq_freq_attr_ro(amd_pstate_highest_perf
);
1044 cpufreq_freq_attr_rw(energy_performance_preference
);
1045 cpufreq_freq_attr_ro(energy_performance_available_preferences
);
1046 static DEVICE_ATTR_RW(status
);
1048 static struct freq_attr
*amd_pstate_attr
[] = {
1049 &amd_pstate_max_freq
,
1050 &amd_pstate_lowest_nonlinear_freq
,
1051 &amd_pstate_highest_perf
,
1055 static struct freq_attr
*amd_pstate_epp_attr
[] = {
1056 &amd_pstate_max_freq
,
1057 &amd_pstate_lowest_nonlinear_freq
,
1058 &amd_pstate_highest_perf
,
1059 &energy_performance_preference
,
1060 &energy_performance_available_preferences
,
1064 static struct attribute
*pstate_global_attributes
[] = {
1065 &dev_attr_status
.attr
,
1069 static const struct attribute_group amd_pstate_global_attr_group
= {
1070 .name
= "amd_pstate",
1071 .attrs
= pstate_global_attributes
,
1074 static bool amd_pstate_acpi_pm_profile_server(void)
1076 switch (acpi_gbl_FADT
.preferred_profile
) {
1077 case PM_ENTERPRISE_SERVER
:
1078 case PM_SOHO_SERVER
:
1079 case PM_PERFORMANCE_SERVER
:
1085 static bool amd_pstate_acpi_pm_profile_undefined(void)
1087 if (acpi_gbl_FADT
.preferred_profile
== PM_UNSPECIFIED
)
1089 if (acpi_gbl_FADT
.preferred_profile
>= NR_PM_PROFILES
)
1094 static int amd_pstate_epp_cpu_init(struct cpufreq_policy
*policy
)
1096 int min_freq
, max_freq
, nominal_freq
, lowest_nonlinear_freq
, ret
;
1097 struct amd_cpudata
*cpudata
;
1102 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1103 * which is ideal for initialization process.
1105 amd_perf_ctl_reset(policy
->cpu
);
1106 dev
= get_cpu_device(policy
->cpu
);
1110 cpudata
= kzalloc(sizeof(*cpudata
), GFP_KERNEL
);
1114 cpudata
->cpu
= policy
->cpu
;
1115 cpudata
->epp_policy
= 0;
1117 ret
= amd_pstate_init_perf(cpudata
);
1121 min_freq
= amd_get_min_freq(cpudata
);
1122 max_freq
= amd_get_max_freq(cpudata
);
1123 nominal_freq
= amd_get_nominal_freq(cpudata
);
1124 lowest_nonlinear_freq
= amd_get_lowest_nonlinear_freq(cpudata
);
1125 if (min_freq
< 0 || max_freq
< 0 || min_freq
> max_freq
) {
1126 dev_err(dev
, "min_freq(%d) or max_freq(%d) value is incorrect\n",
1127 min_freq
, max_freq
);
1132 policy
->cpuinfo
.min_freq
= min_freq
;
1133 policy
->cpuinfo
.max_freq
= max_freq
;
1134 /* It will be updated by governor */
1135 policy
->cur
= policy
->cpuinfo
.min_freq
;
1137 /* Initial processor data capability frequencies */
1138 cpudata
->max_freq
= max_freq
;
1139 cpudata
->min_freq
= min_freq
;
1140 cpudata
->nominal_freq
= nominal_freq
;
1141 cpudata
->lowest_nonlinear_freq
= lowest_nonlinear_freq
;
1143 policy
->driver_data
= cpudata
;
1145 cpudata
->epp_cached
= amd_pstate_get_epp(cpudata
, 0);
1147 policy
->min
= policy
->cpuinfo
.min_freq
;
1148 policy
->max
= policy
->cpuinfo
.max_freq
;
1151 * Set the policy to provide a valid fallback value in case
1152 * the default cpufreq governor is neither powersave nor performance.
1154 if (amd_pstate_acpi_pm_profile_server() ||
1155 amd_pstate_acpi_pm_profile_undefined())
1156 policy
->policy
= CPUFREQ_POLICY_PERFORMANCE
;
1158 policy
->policy
= CPUFREQ_POLICY_POWERSAVE
;
1160 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
1161 ret
= rdmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_REQ
, &value
);
1164 WRITE_ONCE(cpudata
->cppc_req_cached
, value
);
1166 ret
= rdmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_CAP1
, &value
);
1169 WRITE_ONCE(cpudata
->cppc_cap1_cached
, value
);
1171 amd_pstate_boost_init(cpudata
);
1180 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy
*policy
)
1182 pr_debug("CPU %d exiting\n", policy
->cpu
);
1186 static void amd_pstate_epp_init(unsigned int cpu
)
1188 struct cpufreq_policy
*policy
= cpufreq_cpu_get(cpu
);
1189 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1190 u32 max_perf
, min_perf
;
1194 max_perf
= READ_ONCE(cpudata
->highest_perf
);
1195 min_perf
= READ_ONCE(cpudata
->lowest_perf
);
1197 value
= READ_ONCE(cpudata
->cppc_req_cached
);
1199 if (cpudata
->policy
== CPUFREQ_POLICY_PERFORMANCE
)
1200 min_perf
= max_perf
;
1202 /* Initial min/max values for CPPC Performance Controls Register */
1203 value
&= ~AMD_CPPC_MIN_PERF(~0L);
1204 value
|= AMD_CPPC_MIN_PERF(min_perf
);
1206 value
&= ~AMD_CPPC_MAX_PERF(~0L);
1207 value
|= AMD_CPPC_MAX_PERF(max_perf
);
1209 /* CPPC EPP feature require to set zero to the desire perf bit */
1210 value
&= ~AMD_CPPC_DES_PERF(~0L);
1211 value
|= AMD_CPPC_DES_PERF(0);
1213 if (cpudata
->epp_policy
== cpudata
->policy
)
1216 cpudata
->epp_policy
= cpudata
->policy
;
1218 /* Get BIOS pre-defined epp value */
1219 epp
= amd_pstate_get_epp(cpudata
, value
);
1222 * This return value can only be negative for shared_memory
1223 * systems where EPP register read/write not supported.
1228 if (cpudata
->policy
== CPUFREQ_POLICY_PERFORMANCE
)
1231 /* Set initial EPP value */
1232 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
1233 value
&= ~GENMASK_ULL(31, 24);
1234 value
|= (u64
)epp
<< 24;
1237 WRITE_ONCE(cpudata
->cppc_req_cached
, value
);
1238 amd_pstate_set_epp(cpudata
, epp
);
1240 cpufreq_cpu_put(policy
);
1243 static int amd_pstate_epp_set_policy(struct cpufreq_policy
*policy
)
1245 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1247 if (!policy
->cpuinfo
.max_freq
)
1250 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1251 policy
->cpuinfo
.max_freq
, policy
->max
);
1253 cpudata
->policy
= policy
->policy
;
1255 amd_pstate_epp_init(policy
->cpu
);
1260 static void amd_pstate_epp_reenable(struct amd_cpudata
*cpudata
)
1262 struct cppc_perf_ctrls perf_ctrls
;
1263 u64 value
, max_perf
;
1266 ret
= amd_pstate_enable(true);
1268 pr_err("failed to enable amd pstate during resume, return %d\n", ret
);
1270 value
= READ_ONCE(cpudata
->cppc_req_cached
);
1271 max_perf
= READ_ONCE(cpudata
->highest_perf
);
1273 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
1274 wrmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_REQ
, value
);
1276 perf_ctrls
.max_perf
= max_perf
;
1277 perf_ctrls
.energy_perf
= AMD_CPPC_ENERGY_PERF_PREF(cpudata
->epp_cached
);
1278 cppc_set_perf(cpudata
->cpu
, &perf_ctrls
);
1282 static int amd_pstate_epp_cpu_online(struct cpufreq_policy
*policy
)
1284 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1286 pr_debug("AMD CPU Core %d going online\n", cpudata
->cpu
);
1288 if (cppc_state
== AMD_PSTATE_ACTIVE
) {
1289 amd_pstate_epp_reenable(cpudata
);
1290 cpudata
->suspended
= false;
1296 static void amd_pstate_epp_offline(struct cpufreq_policy
*policy
)
1298 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1299 struct cppc_perf_ctrls perf_ctrls
;
1303 min_perf
= READ_ONCE(cpudata
->lowest_perf
);
1304 value
= READ_ONCE(cpudata
->cppc_req_cached
);
1306 mutex_lock(&amd_pstate_limits_lock
);
1307 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
1308 cpudata
->epp_policy
= CPUFREQ_POLICY_UNKNOWN
;
1310 /* Set max perf same as min perf */
1311 value
&= ~AMD_CPPC_MAX_PERF(~0L);
1312 value
|= AMD_CPPC_MAX_PERF(min_perf
);
1313 value
&= ~AMD_CPPC_MIN_PERF(~0L);
1314 value
|= AMD_CPPC_MIN_PERF(min_perf
);
1315 wrmsrl_on_cpu(cpudata
->cpu
, MSR_AMD_CPPC_REQ
, value
);
1317 perf_ctrls
.desired_perf
= 0;
1318 perf_ctrls
.max_perf
= min_perf
;
1319 perf_ctrls
.energy_perf
= AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE
);
1320 cppc_set_perf(cpudata
->cpu
, &perf_ctrls
);
1322 mutex_unlock(&amd_pstate_limits_lock
);
1325 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy
*policy
)
1327 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1329 pr_debug("AMD CPU Core %d going offline\n", cpudata
->cpu
);
1331 if (cpudata
->suspended
)
1334 if (cppc_state
== AMD_PSTATE_ACTIVE
)
1335 amd_pstate_epp_offline(policy
);
1340 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data
*policy
)
1342 cpufreq_verify_within_cpu_limits(policy
);
1343 pr_debug("policy_max =%d, policy_min=%d\n", policy
->max
, policy
->min
);
1347 static int amd_pstate_epp_suspend(struct cpufreq_policy
*policy
)
1349 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1352 /* avoid suspending when EPP is not enabled */
1353 if (cppc_state
!= AMD_PSTATE_ACTIVE
)
1356 /* set this flag to avoid setting core offline*/
1357 cpudata
->suspended
= true;
1359 /* disable CPPC in lowlevel firmware */
1360 ret
= amd_pstate_enable(false);
1362 pr_err("failed to suspend, return %d\n", ret
);
1367 static int amd_pstate_epp_resume(struct cpufreq_policy
*policy
)
1369 struct amd_cpudata
*cpudata
= policy
->driver_data
;
1371 if (cpudata
->suspended
) {
1372 mutex_lock(&amd_pstate_limits_lock
);
1374 /* enable amd pstate from suspend state*/
1375 amd_pstate_epp_reenable(cpudata
);
1377 mutex_unlock(&amd_pstate_limits_lock
);
1379 cpudata
->suspended
= false;
1385 static struct cpufreq_driver amd_pstate_driver
= {
1386 .flags
= CPUFREQ_CONST_LOOPS
| CPUFREQ_NEED_UPDATE_LIMITS
,
1387 .verify
= amd_pstate_verify
,
1388 .target
= amd_pstate_target
,
1389 .fast_switch
= amd_pstate_fast_switch
,
1390 .init
= amd_pstate_cpu_init
,
1391 .exit
= amd_pstate_cpu_exit
,
1392 .suspend
= amd_pstate_cpu_suspend
,
1393 .resume
= amd_pstate_cpu_resume
,
1394 .set_boost
= amd_pstate_set_boost
,
1395 .name
= "amd-pstate",
1396 .attr
= amd_pstate_attr
,
1399 static struct cpufreq_driver amd_pstate_epp_driver
= {
1400 .flags
= CPUFREQ_CONST_LOOPS
,
1401 .verify
= amd_pstate_epp_verify_policy
,
1402 .setpolicy
= amd_pstate_epp_set_policy
,
1403 .init
= amd_pstate_epp_cpu_init
,
1404 .exit
= amd_pstate_epp_cpu_exit
,
1405 .offline
= amd_pstate_epp_cpu_offline
,
1406 .online
= amd_pstate_epp_cpu_online
,
1407 .suspend
= amd_pstate_epp_suspend
,
1408 .resume
= amd_pstate_epp_resume
,
1409 .name
= "amd-pstate-epp",
1410 .attr
= amd_pstate_epp_attr
,
1413 static int __init
amd_pstate_set_driver(int mode_idx
)
1415 if (mode_idx
>= AMD_PSTATE_DISABLE
&& mode_idx
< AMD_PSTATE_MAX
) {
1416 cppc_state
= mode_idx
;
1417 if (cppc_state
== AMD_PSTATE_DISABLE
)
1418 pr_info("driver is explicitly disabled\n");
1420 if (cppc_state
== AMD_PSTATE_ACTIVE
)
1421 current_pstate_driver
= &amd_pstate_epp_driver
;
1423 if (cppc_state
== AMD_PSTATE_PASSIVE
|| cppc_state
== AMD_PSTATE_GUIDED
)
1424 current_pstate_driver
= &amd_pstate_driver
;
1432 static int __init
amd_pstate_init(void)
1434 struct device
*dev_root
;
1437 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
)
1440 if (!acpi_cpc_valid()) {
1441 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
1445 /* don't keep reloading if cpufreq_driver exists */
1446 if (cpufreq_get_current_driver())
1449 switch (cppc_state
) {
1450 case AMD_PSTATE_UNDEFINED
:
1451 /* Disable on the following configs by default:
1452 * 1. Undefined platforms
1453 * 2. Server platforms
1454 * 3. Shared memory designs
1456 if (amd_pstate_acpi_pm_profile_undefined() ||
1457 amd_pstate_acpi_pm_profile_server() ||
1458 !boot_cpu_has(X86_FEATURE_CPPC
)) {
1459 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1462 ret
= amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE
);
1466 case AMD_PSTATE_DISABLE
:
1468 case AMD_PSTATE_PASSIVE
:
1469 case AMD_PSTATE_ACTIVE
:
1470 case AMD_PSTATE_GUIDED
:
1476 /* capability check */
1477 if (boot_cpu_has(X86_FEATURE_CPPC
)) {
1478 pr_debug("AMD CPPC MSR based functionality is supported\n");
1479 if (cppc_state
!= AMD_PSTATE_ACTIVE
)
1480 current_pstate_driver
->adjust_perf
= amd_pstate_adjust_perf
;
1482 pr_debug("AMD CPPC shared memory based functionality is supported\n");
1483 static_call_update(amd_pstate_enable
, cppc_enable
);
1484 static_call_update(amd_pstate_init_perf
, cppc_init_perf
);
1485 static_call_update(amd_pstate_update_perf
, cppc_update_perf
);
1488 /* enable amd pstate feature */
1489 ret
= amd_pstate_enable(true);
1491 pr_err("failed to enable with return %d\n", ret
);
1495 ret
= cpufreq_register_driver(current_pstate_driver
);
1497 pr_err("failed to register with return %d\n", ret
);
1499 dev_root
= bus_get_dev_root(&cpu_subsys
);
1501 ret
= sysfs_create_group(&dev_root
->kobj
, &amd_pstate_global_attr_group
);
1502 put_device(dev_root
);
1504 pr_err("sysfs attribute export failed with error %d.\n", ret
);
1505 goto global_attr_free
;
1512 cpufreq_unregister_driver(current_pstate_driver
);
1515 device_initcall(amd_pstate_init
);
1517 static int __init
amd_pstate_param(char *str
)
1526 mode_idx
= get_mode_idx_from_str(str
, size
);
1528 return amd_pstate_set_driver(mode_idx
);
1530 early_param("amd_pstate", amd_pstate_param
);
1532 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
1533 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");