1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 #include <linux/arm-smccc.h>
8 #include <linux/preempt.h>
9 #include <linux/kvm_host.h>
10 #include <linux/uaccess.h>
11 #include <linux/wait.h>
13 #include <asm/cputype.h>
14 #include <asm/kvm_emulate.h>
16 #include <kvm/arm_psci.h>
17 #include <kvm/arm_hypercalls.h>
20 * This is an implementation of the Power State Coordination Interface
21 * as described in ARM document number ARM DEN 0022A.
24 #define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
26 static unsigned long psci_affinity_mask(unsigned long affinity_level
)
28 if (affinity_level
<= 3)
29 return MPIDR_HWID_BITMASK
& AFFINITY_MASK(affinity_level
);
34 static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu
*vcpu
)
37 * NOTE: For simplicity, we make VCPU suspend emulation to be
38 * same-as WFI (Wait-for-interrupt) emulation.
40 * This means for KVM the wakeup events are interrupts and
41 * this is consistent with intended use of StateID as described
42 * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A).
44 * Further, we also treat power-down request to be same as
45 * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2
46 * specification (ARM DEN 0022A). This means all suspend states
47 * for KVM will preserve the register state.
50 kvm_clear_request(KVM_REQ_UNHALT
, vcpu
);
52 return PSCI_RET_SUCCESS
;
55 static void kvm_psci_vcpu_off(struct kvm_vcpu
*vcpu
)
57 vcpu
->arch
.power_off
= true;
58 kvm_make_request(KVM_REQ_SLEEP
, vcpu
);
62 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu
*source_vcpu
)
64 struct vcpu_reset_state
*reset_state
;
65 struct kvm
*kvm
= source_vcpu
->kvm
;
66 struct kvm_vcpu
*vcpu
= NULL
;
69 cpu_id
= smccc_get_arg1(source_vcpu
) & MPIDR_HWID_BITMASK
;
70 if (vcpu_mode_is_32bit(source_vcpu
))
73 vcpu
= kvm_mpidr_to_vcpu(kvm
, cpu_id
);
76 * Make sure the caller requested a valid CPU and that the CPU is
80 return PSCI_RET_INVALID_PARAMS
;
81 if (!vcpu
->arch
.power_off
) {
82 if (kvm_psci_version(source_vcpu
, kvm
) != KVM_ARM_PSCI_0_1
)
83 return PSCI_RET_ALREADY_ON
;
85 return PSCI_RET_INVALID_PARAMS
;
88 reset_state
= &vcpu
->arch
.reset_state
;
90 reset_state
->pc
= smccc_get_arg2(source_vcpu
);
92 /* Propagate caller endianness */
93 reset_state
->be
= kvm_vcpu_is_be(source_vcpu
);
96 * NOTE: We always update r0 (or x0) because for PSCI v0.1
97 * the general puspose registers are undefined upon CPU_ON.
99 reset_state
->r0
= smccc_get_arg3(source_vcpu
);
101 WRITE_ONCE(reset_state
->reset
, true);
102 kvm_make_request(KVM_REQ_VCPU_RESET
, vcpu
);
105 * Make sure the reset request is observed if the change to
106 * power_state is observed.
110 vcpu
->arch
.power_off
= false;
111 kvm_vcpu_wake_up(vcpu
);
113 return PSCI_RET_SUCCESS
;
116 static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu
*vcpu
)
118 int i
, matching_cpus
= 0;
120 unsigned long target_affinity
;
121 unsigned long target_affinity_mask
;
122 unsigned long lowest_affinity_level
;
123 struct kvm
*kvm
= vcpu
->kvm
;
124 struct kvm_vcpu
*tmp
;
126 target_affinity
= smccc_get_arg1(vcpu
);
127 lowest_affinity_level
= smccc_get_arg2(vcpu
);
129 /* Determine target affinity mask */
130 target_affinity_mask
= psci_affinity_mask(lowest_affinity_level
);
131 if (!target_affinity_mask
)
132 return PSCI_RET_INVALID_PARAMS
;
134 /* Ignore other bits of target affinity */
135 target_affinity
&= target_affinity_mask
;
138 * If one or more VCPU matching target affinity are running
141 kvm_for_each_vcpu(i
, tmp
, kvm
) {
142 mpidr
= kvm_vcpu_get_mpidr_aff(tmp
);
143 if ((mpidr
& target_affinity_mask
) == target_affinity
) {
145 if (!tmp
->arch
.power_off
)
146 return PSCI_0_2_AFFINITY_LEVEL_ON
;
151 return PSCI_RET_INVALID_PARAMS
;
153 return PSCI_0_2_AFFINITY_LEVEL_OFF
;
156 static void kvm_prepare_system_event(struct kvm_vcpu
*vcpu
, u32 type
)
159 struct kvm_vcpu
*tmp
;
162 * The KVM ABI specifies that a system event exit may call KVM_RUN
163 * again and may perform shutdown/reboot at a later time that when the
164 * actual request is made. Since we are implementing PSCI and a
165 * caller of PSCI reboot and shutdown expects that the system shuts
166 * down or reboots immediately, let's make sure that VCPUs are not run
167 * after this call is handled and before the VCPUs have been
170 kvm_for_each_vcpu(i
, tmp
, vcpu
->kvm
)
171 tmp
->arch
.power_off
= true;
172 kvm_make_all_cpus_request(vcpu
->kvm
, KVM_REQ_SLEEP
);
174 memset(&vcpu
->run
->system_event
, 0, sizeof(vcpu
->run
->system_event
));
175 vcpu
->run
->system_event
.type
= type
;
176 vcpu
->run
->exit_reason
= KVM_EXIT_SYSTEM_EVENT
;
179 static void kvm_psci_system_off(struct kvm_vcpu
*vcpu
)
181 kvm_prepare_system_event(vcpu
, KVM_SYSTEM_EVENT_SHUTDOWN
);
184 static void kvm_psci_system_reset(struct kvm_vcpu
*vcpu
)
186 kvm_prepare_system_event(vcpu
, KVM_SYSTEM_EVENT_RESET
);
189 static int kvm_psci_0_2_call(struct kvm_vcpu
*vcpu
)
191 struct kvm
*kvm
= vcpu
->kvm
;
192 u32 psci_fn
= smccc_get_function(vcpu
);
197 case PSCI_0_2_FN_PSCI_VERSION
:
199 * Bits[31:16] = Major Version = 0
200 * Bits[15:0] = Minor Version = 2
202 val
= KVM_ARM_PSCI_0_2
;
204 case PSCI_0_2_FN_CPU_SUSPEND
:
205 case PSCI_0_2_FN64_CPU_SUSPEND
:
206 val
= kvm_psci_vcpu_suspend(vcpu
);
208 case PSCI_0_2_FN_CPU_OFF
:
209 kvm_psci_vcpu_off(vcpu
);
210 val
= PSCI_RET_SUCCESS
;
212 case PSCI_0_2_FN_CPU_ON
:
213 case PSCI_0_2_FN64_CPU_ON
:
214 mutex_lock(&kvm
->lock
);
215 val
= kvm_psci_vcpu_on(vcpu
);
216 mutex_unlock(&kvm
->lock
);
218 case PSCI_0_2_FN_AFFINITY_INFO
:
219 case PSCI_0_2_FN64_AFFINITY_INFO
:
220 val
= kvm_psci_vcpu_affinity_info(vcpu
);
222 case PSCI_0_2_FN_MIGRATE_INFO_TYPE
:
224 * Trusted OS is MP hence does not require migration
226 * Trusted OS is not present
228 val
= PSCI_0_2_TOS_MP
;
230 case PSCI_0_2_FN_SYSTEM_OFF
:
231 kvm_psci_system_off(vcpu
);
233 * We should'nt be going back to guest VCPU after
234 * receiving SYSTEM_OFF request.
236 * If user space accidently/deliberately resumes
237 * guest VCPU after SYSTEM_OFF request then guest
238 * VCPU should see internal failure from PSCI return
239 * value. To achieve this, we preload r0 (or x0) with
240 * PSCI return value INTERNAL_FAILURE.
242 val
= PSCI_RET_INTERNAL_FAILURE
;
245 case PSCI_0_2_FN_SYSTEM_RESET
:
246 kvm_psci_system_reset(vcpu
);
248 * Same reason as SYSTEM_OFF for preloading r0 (or x0)
249 * with PSCI return value INTERNAL_FAILURE.
251 val
= PSCI_RET_INTERNAL_FAILURE
;
255 val
= PSCI_RET_NOT_SUPPORTED
;
259 smccc_set_retval(vcpu
, val
, 0, 0, 0);
263 static int kvm_psci_1_0_call(struct kvm_vcpu
*vcpu
)
265 u32 psci_fn
= smccc_get_function(vcpu
);
271 case PSCI_0_2_FN_PSCI_VERSION
:
272 val
= KVM_ARM_PSCI_1_0
;
274 case PSCI_1_0_FN_PSCI_FEATURES
:
275 feature
= smccc_get_arg1(vcpu
);
277 case PSCI_0_2_FN_PSCI_VERSION
:
278 case PSCI_0_2_FN_CPU_SUSPEND
:
279 case PSCI_0_2_FN64_CPU_SUSPEND
:
280 case PSCI_0_2_FN_CPU_OFF
:
281 case PSCI_0_2_FN_CPU_ON
:
282 case PSCI_0_2_FN64_CPU_ON
:
283 case PSCI_0_2_FN_AFFINITY_INFO
:
284 case PSCI_0_2_FN64_AFFINITY_INFO
:
285 case PSCI_0_2_FN_MIGRATE_INFO_TYPE
:
286 case PSCI_0_2_FN_SYSTEM_OFF
:
287 case PSCI_0_2_FN_SYSTEM_RESET
:
288 case PSCI_1_0_FN_PSCI_FEATURES
:
289 case ARM_SMCCC_VERSION_FUNC_ID
:
293 val
= PSCI_RET_NOT_SUPPORTED
;
298 return kvm_psci_0_2_call(vcpu
);
301 smccc_set_retval(vcpu
, val
, 0, 0, 0);
305 static int kvm_psci_0_1_call(struct kvm_vcpu
*vcpu
)
307 struct kvm
*kvm
= vcpu
->kvm
;
308 u32 psci_fn
= smccc_get_function(vcpu
);
312 case KVM_PSCI_FN_CPU_OFF
:
313 kvm_psci_vcpu_off(vcpu
);
314 val
= PSCI_RET_SUCCESS
;
316 case KVM_PSCI_FN_CPU_ON
:
317 mutex_lock(&kvm
->lock
);
318 val
= kvm_psci_vcpu_on(vcpu
);
319 mutex_unlock(&kvm
->lock
);
322 val
= PSCI_RET_NOT_SUPPORTED
;
326 smccc_set_retval(vcpu
, val
, 0, 0, 0);
331 * kvm_psci_call - handle PSCI call if r0 value is in range
332 * @vcpu: Pointer to the VCPU struct
334 * Handle PSCI calls from guests through traps from HVC instructions.
335 * The calling convention is similar to SMC calls to the secure world
336 * where the function number is placed in r0.
338 * This function returns: > 0 (success), 0 (success but exit to user
339 * space), and < 0 (errors)
342 * -EINVAL: Unrecognized PSCI function
344 int kvm_psci_call(struct kvm_vcpu
*vcpu
)
346 switch (kvm_psci_version(vcpu
, vcpu
->kvm
)) {
347 case KVM_ARM_PSCI_1_0
:
348 return kvm_psci_1_0_call(vcpu
);
349 case KVM_ARM_PSCI_0_2
:
350 return kvm_psci_0_2_call(vcpu
);
351 case KVM_ARM_PSCI_0_1
:
352 return kvm_psci_0_1_call(vcpu
);
358 int kvm_arm_get_fw_num_regs(struct kvm_vcpu
*vcpu
)
360 return 3; /* PSCI version and two workaround registers */
363 int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu
*vcpu
, u64 __user
*uindices
)
365 if (put_user(KVM_REG_ARM_PSCI_VERSION
, uindices
++))
368 if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1
, uindices
++))
371 if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2
, uindices
++))
377 #define KVM_REG_FEATURE_LEVEL_WIDTH 4
378 #define KVM_REG_FEATURE_LEVEL_MASK (BIT(KVM_REG_FEATURE_LEVEL_WIDTH) - 1)
381 * Convert the workaround level into an easy-to-compare number, where higher
382 * values mean better protection.
384 static int get_kernel_wa_level(u64 regid
)
387 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1
:
388 switch (kvm_arm_harden_branch_predictor()) {
389 case KVM_BP_HARDEN_UNKNOWN
:
390 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL
;
391 case KVM_BP_HARDEN_WA_NEEDED
:
392 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL
;
393 case KVM_BP_HARDEN_NOT_REQUIRED
:
394 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED
;
396 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL
;
397 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2
:
398 switch (kvm_arm_have_ssbd()) {
399 case KVM_SSBD_FORCE_DISABLE
:
400 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL
;
401 case KVM_SSBD_KERNEL
:
402 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL
;
403 case KVM_SSBD_FORCE_ENABLE
:
404 case KVM_SSBD_MITIGATED
:
405 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED
;
406 case KVM_SSBD_UNKNOWN
:
408 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN
;
415 int kvm_arm_get_fw_reg(struct kvm_vcpu
*vcpu
, const struct kvm_one_reg
*reg
)
417 void __user
*uaddr
= (void __user
*)(long)reg
->addr
;
421 case KVM_REG_ARM_PSCI_VERSION
:
422 val
= kvm_psci_version(vcpu
, vcpu
->kvm
);
424 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1
:
425 val
= get_kernel_wa_level(reg
->id
) & KVM_REG_FEATURE_LEVEL_MASK
;
427 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2
:
428 val
= get_kernel_wa_level(reg
->id
) & KVM_REG_FEATURE_LEVEL_MASK
;
430 if (val
== KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL
&&
431 kvm_arm_get_vcpu_workaround_2_flag(vcpu
))
432 val
|= KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED
;
438 if (copy_to_user(uaddr
, &val
, KVM_REG_SIZE(reg
->id
)))
444 int kvm_arm_set_fw_reg(struct kvm_vcpu
*vcpu
, const struct kvm_one_reg
*reg
)
446 void __user
*uaddr
= (void __user
*)(long)reg
->addr
;
450 if (copy_from_user(&val
, uaddr
, KVM_REG_SIZE(reg
->id
)))
454 case KVM_REG_ARM_PSCI_VERSION
:
458 wants_02
= test_bit(KVM_ARM_VCPU_PSCI_0_2
, vcpu
->arch
.features
);
461 case KVM_ARM_PSCI_0_1
:
464 vcpu
->kvm
->arch
.psci_version
= val
;
466 case KVM_ARM_PSCI_0_2
:
467 case KVM_ARM_PSCI_1_0
:
470 vcpu
->kvm
->arch
.psci_version
= val
;
476 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1
:
477 if (val
& ~KVM_REG_FEATURE_LEVEL_MASK
)
480 if (get_kernel_wa_level(reg
->id
) < val
)
485 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2
:
486 if (val
& ~(KVM_REG_FEATURE_LEVEL_MASK
|
487 KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED
))
490 wa_level
= val
& KVM_REG_FEATURE_LEVEL_MASK
;
492 if (get_kernel_wa_level(reg
->id
) < wa_level
)
495 /* The enabled bit must not be set unless the level is AVAIL. */
496 if (wa_level
!= KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL
&&
500 /* Are we finished or do we need to check the enable bit ? */
501 if (kvm_arm_have_ssbd() != KVM_SSBD_KERNEL
)
505 * If this kernel supports the workaround to be switched on
506 * or off, make sure it matches the requested setting.
509 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL
:
510 kvm_arm_set_vcpu_workaround_2_flag(vcpu
,
511 val
& KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED
);
513 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED
:
514 kvm_arm_set_vcpu_workaround_2_flag(vcpu
, true);