]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
2f4a07c5 MZ |
2 | /* |
3 | * Copyright (C) 2012,2013 - ARM Ltd | |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
5 | * | |
6 | * Derived from arch/arm/kvm/guest.c: | |
7 | * Copyright (C) 2012 - Virtual Open Systems and Columbia University | |
8 | * Author: Christoffer Dall <c.dall@virtualopensystems.com> | |
2f4a07c5 MZ |
9 | */ |
10 | ||
e1c9c983 | 11 | #include <linux/bits.h> |
2f4a07c5 MZ |
12 | #include <linux/errno.h> |
13 | #include <linux/err.h> | |
e1c9c983 | 14 | #include <linux/nospec.h> |
2f4a07c5 MZ |
15 | #include <linux/kvm_host.h> |
16 | #include <linux/module.h> | |
be25bbb3 | 17 | #include <linux/stddef.h> |
dc52f31a | 18 | #include <linux/string.h> |
2f4a07c5 MZ |
19 | #include <linux/vmalloc.h> |
20 | #include <linux/fs.h> | |
85bd0ba1 | 21 | #include <kvm/arm_psci.h> |
2f4a07c5 | 22 | #include <asm/cputype.h> |
7c0f6ba6 | 23 | #include <linux/uaccess.h> |
e1c9c983 | 24 | #include <asm/fpsimd.h> |
2f4a07c5 | 25 | #include <asm/kvm.h> |
2f4a07c5 MZ |
26 | #include <asm/kvm_emulate.h> |
27 | #include <asm/kvm_coproc.h> | |
e1c9c983 | 28 | #include <asm/sigcontext.h> |
2f4a07c5 | 29 | |
eef8c85a AB |
30 | #include "trace.h" |
31 | ||
b19e6892 AT |
32 | #define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM } |
33 | #define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU } | |
34 | ||
2f4a07c5 | 35 | struct kvm_stats_debugfs_item debugfs_entries[] = { |
01d035d7 CB |
36 | VCPU_STAT(halt_successful_poll), |
37 | VCPU_STAT(halt_attempted_poll), | |
38 | VCPU_STAT(halt_poll_invalid), | |
39 | VCPU_STAT(halt_wakeup), | |
b19e6892 AT |
40 | VCPU_STAT(hvc_exit_stat), |
41 | VCPU_STAT(wfe_exit_stat), | |
42 | VCPU_STAT(wfi_exit_stat), | |
43 | VCPU_STAT(mmio_exit_user), | |
44 | VCPU_STAT(mmio_exit_kernel), | |
45 | VCPU_STAT(exits), | |
2f4a07c5 MZ |
46 | { NULL } |
47 | }; | |
48 | ||
8c86dfe3 DM |
49 | static bool core_reg_offset_is_vreg(u64 off) |
50 | { | |
51 | return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && | |
52 | off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr); | |
53 | } | |
54 | ||
2f4a07c5 MZ |
55 | static u64 core_reg_offset_from_id(u64 id) |
56 | { | |
57 | return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); | |
58 | } | |
59 | ||
df205b5c | 60 | static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off) |
d26c25a9 | 61 | { |
d26c25a9 DM |
62 | int size; |
63 | ||
64 | switch (off) { | |
65 | case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... | |
66 | KVM_REG_ARM_CORE_REG(regs.regs[30]): | |
67 | case KVM_REG_ARM_CORE_REG(regs.sp): | |
68 | case KVM_REG_ARM_CORE_REG(regs.pc): | |
69 | case KVM_REG_ARM_CORE_REG(regs.pstate): | |
70 | case KVM_REG_ARM_CORE_REG(sp_el1): | |
71 | case KVM_REG_ARM_CORE_REG(elr_el1): | |
72 | case KVM_REG_ARM_CORE_REG(spsr[0]) ... | |
73 | KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): | |
74 | size = sizeof(__u64); | |
75 | break; | |
76 | ||
77 | case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... | |
78 | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): | |
79 | size = sizeof(__uint128_t); | |
80 | break; | |
81 | ||
82 | case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): | |
83 | case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): | |
84 | size = sizeof(__u32); | |
85 | break; | |
86 | ||
87 | default: | |
88 | return -EINVAL; | |
89 | } | |
90 | ||
df205b5c | 91 | if (!IS_ALIGNED(off, size / sizeof(__u32))) |
8c86dfe3 | 92 | return -EINVAL; |
d26c25a9 | 93 | |
8c86dfe3 DM |
94 | /* |
95 | * The KVM_REG_ARM64_SVE regs must be used instead of | |
96 | * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on | |
97 | * SVE-enabled vcpus: | |
98 | */ | |
99 | if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off)) | |
100 | return -EINVAL; | |
101 | ||
df205b5c DM |
102 | return size; |
103 | } | |
104 | ||
105 | static int validate_core_offset(const struct kvm_vcpu *vcpu, | |
106 | const struct kvm_one_reg *reg) | |
107 | { | |
108 | u64 off = core_reg_offset_from_id(reg->id); | |
109 | int size = core_reg_size_from_offset(vcpu, off); | |
110 | ||
111 | if (size < 0) | |
112 | return -EINVAL; | |
113 | ||
114 | if (KVM_REG_SIZE(reg->id) != size) | |
115 | return -EINVAL; | |
116 | ||
8c86dfe3 | 117 | return 0; |
d26c25a9 DM |
118 | } |
119 | ||
2f4a07c5 MZ |
120 | static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) |
121 | { | |
122 | /* | |
123 | * Because the kvm_regs structure is a mix of 32, 64 and | |
124 | * 128bit fields, we index it as if it was a 32bit | |
125 | * array. Hence below, nr_regs is the number of entries, and | |
126 | * off the index in the "array". | |
127 | */ | |
128 | __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; | |
129 | struct kvm_regs *regs = vcpu_gp_regs(vcpu); | |
130 | int nr_regs = sizeof(*regs) / sizeof(__u32); | |
131 | u32 off; | |
132 | ||
133 | /* Our ID is an index into the kvm_regs struct. */ | |
134 | off = core_reg_offset_from_id(reg->id); | |
135 | if (off >= nr_regs || | |
136 | (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) | |
137 | return -ENOENT; | |
138 | ||
8c86dfe3 | 139 | if (validate_core_offset(vcpu, reg)) |
d26c25a9 DM |
140 | return -EINVAL; |
141 | ||
2f4a07c5 MZ |
142 | if (copy_to_user(uaddr, ((u32 *)regs) + off, KVM_REG_SIZE(reg->id))) |
143 | return -EFAULT; | |
144 | ||
145 | return 0; | |
146 | } | |
147 | ||
148 | static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
149 | { | |
150 | __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; | |
151 | struct kvm_regs *regs = vcpu_gp_regs(vcpu); | |
152 | int nr_regs = sizeof(*regs) / sizeof(__u32); | |
153 | __uint128_t tmp; | |
154 | void *valp = &tmp; | |
155 | u64 off; | |
156 | int err = 0; | |
157 | ||
158 | /* Our ID is an index into the kvm_regs struct. */ | |
159 | off = core_reg_offset_from_id(reg->id); | |
160 | if (off >= nr_regs || | |
161 | (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) | |
162 | return -ENOENT; | |
163 | ||
8c86dfe3 | 164 | if (validate_core_offset(vcpu, reg)) |
d26c25a9 DM |
165 | return -EINVAL; |
166 | ||
2f4a07c5 MZ |
167 | if (KVM_REG_SIZE(reg->id) > sizeof(tmp)) |
168 | return -EINVAL; | |
169 | ||
170 | if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) { | |
171 | err = -EFAULT; | |
172 | goto out; | |
173 | } | |
174 | ||
175 | if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) { | |
2a3f9345 | 176 | u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; |
2f4a07c5 | 177 | switch (mode) { |
256c0960 | 178 | case PSR_AA32_MODE_USR: |
2a3f9345 MZ |
179 | if (!system_supports_32bit_el0()) |
180 | return -EINVAL; | |
181 | break; | |
256c0960 MR |
182 | case PSR_AA32_MODE_FIQ: |
183 | case PSR_AA32_MODE_IRQ: | |
184 | case PSR_AA32_MODE_SVC: | |
185 | case PSR_AA32_MODE_ABT: | |
186 | case PSR_AA32_MODE_UND: | |
2a3f9345 MZ |
187 | if (!vcpu_el1_is_32bit(vcpu)) |
188 | return -EINVAL; | |
189 | break; | |
2f4a07c5 MZ |
190 | case PSR_MODE_EL0t: |
191 | case PSR_MODE_EL1t: | |
192 | case PSR_MODE_EL1h: | |
2a3f9345 MZ |
193 | if (vcpu_el1_is_32bit(vcpu)) |
194 | return -EINVAL; | |
2f4a07c5 MZ |
195 | break; |
196 | default: | |
197 | err = -EINVAL; | |
198 | goto out; | |
199 | } | |
200 | } | |
201 | ||
202 | memcpy((u32 *)regs + off, valp, KVM_REG_SIZE(reg->id)); | |
0225fd5e MZ |
203 | |
204 | if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) { | |
205 | int i; | |
206 | ||
207 | for (i = 0; i < 16; i++) | |
208 | *vcpu_reg32(vcpu, i) = (u32)*vcpu_reg32(vcpu, i); | |
209 | } | |
2f4a07c5 MZ |
210 | out: |
211 | return err; | |
212 | } | |
213 | ||
9033bba4 DM |
214 | #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) |
215 | #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) | |
e644fa18 | 216 | #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) |
9033bba4 DM |
217 | |
218 | static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
219 | { | |
220 | unsigned int max_vq, vq; | |
4bd774e5 | 221 | u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; |
9033bba4 | 222 | |
52110aa9 DM |
223 | if (!vcpu_has_sve(vcpu)) |
224 | return -ENOENT; | |
225 | ||
9033bba4 DM |
226 | if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) |
227 | return -EINVAL; | |
228 | ||
229 | memset(vqs, 0, sizeof(vqs)); | |
230 | ||
231 | max_vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); | |
232 | for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) | |
233 | if (sve_vq_available(vq)) | |
234 | vqs[vq_word(vq)] |= vq_mask(vq); | |
235 | ||
236 | if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs))) | |
237 | return -EFAULT; | |
238 | ||
239 | return 0; | |
240 | } | |
241 | ||
242 | static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
243 | { | |
244 | unsigned int max_vq, vq; | |
4bd774e5 | 245 | u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; |
9033bba4 | 246 | |
52110aa9 DM |
247 | if (!vcpu_has_sve(vcpu)) |
248 | return -ENOENT; | |
249 | ||
9033bba4 DM |
250 | if (kvm_arm_vcpu_sve_finalized(vcpu)) |
251 | return -EPERM; /* too late! */ | |
252 | ||
253 | if (WARN_ON(vcpu->arch.sve_state)) | |
254 | return -EINVAL; | |
255 | ||
256 | if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs))) | |
257 | return -EFAULT; | |
258 | ||
259 | max_vq = 0; | |
260 | for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq) | |
0c529ff7 | 261 | if (vq_present(vqs, vq)) |
9033bba4 DM |
262 | max_vq = vq; |
263 | ||
264 | if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) | |
265 | return -EINVAL; | |
266 | ||
ecfb6ed4 DM |
267 | /* |
268 | * Vector lengths supported by the host can't currently be | |
269 | * hidden from the guest individually: instead we can only set a | |
270 | * maxmium via ZCR_EL2.LEN. So, make sure the available vector | |
271 | * lengths match the set requested exactly up to the requested | |
272 | * maximum: | |
273 | */ | |
9033bba4 | 274 | for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) |
0c529ff7 | 275 | if (vq_present(vqs, vq) != sve_vq_available(vq)) |
9033bba4 DM |
276 | return -EINVAL; |
277 | ||
278 | /* Can't run with no vector lengths at all: */ | |
279 | if (max_vq < SVE_VQ_MIN) | |
280 | return -EINVAL; | |
281 | ||
282 | /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ | |
283 | vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); | |
284 | ||
285 | return 0; | |
286 | } | |
287 | ||
e1c9c983 DM |
288 | #define SVE_REG_SLICE_SHIFT 0 |
289 | #define SVE_REG_SLICE_BITS 5 | |
290 | #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS) | |
291 | #define SVE_REG_ID_BITS 5 | |
292 | ||
293 | #define SVE_REG_SLICE_MASK \ | |
294 | GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \ | |
295 | SVE_REG_SLICE_SHIFT) | |
296 | #define SVE_REG_ID_MASK \ | |
297 | GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT) | |
298 | ||
299 | #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS) | |
300 | ||
301 | #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0)) | |
302 | #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0)) | |
303 | ||
8e3c54c8 | 304 | /* |
f8d4635a DM |
305 | * Number of register slices required to cover each whole SVE register. |
306 | * NOTE: Only the first slice every exists, for now. | |
307 | * If you are tempted to modify this, you must also rework sve_reg_to_region() | |
308 | * to match: | |
8e3c54c8 DM |
309 | */ |
310 | #define vcpu_sve_slices(vcpu) 1 | |
311 | ||
e1c9c983 DM |
312 | /* Bounds of a single SVE register slice within vcpu->arch.sve_state */ |
313 | struct sve_state_reg_region { | |
314 | unsigned int koffset; /* offset into sve_state in kernel memory */ | |
315 | unsigned int klen; /* length in kernel memory */ | |
316 | unsigned int upad; /* extra trailing padding in user memory */ | |
317 | }; | |
318 | ||
52110aa9 DM |
319 | /* |
320 | * Validate SVE register ID and get sanitised bounds for user/kernel SVE | |
321 | * register copy | |
322 | */ | |
e1c9c983 DM |
323 | static int sve_reg_to_region(struct sve_state_reg_region *region, |
324 | struct kvm_vcpu *vcpu, | |
325 | const struct kvm_one_reg *reg) | |
326 | { | |
327 | /* reg ID ranges for Z- registers */ | |
328 | const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0); | |
329 | const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1, | |
330 | SVE_NUM_SLICES - 1); | |
331 | ||
332 | /* reg ID ranges for P- registers and FFR (which are contiguous) */ | |
333 | const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0); | |
334 | const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1); | |
335 | ||
336 | unsigned int vq; | |
337 | unsigned int reg_num; | |
338 | ||
339 | unsigned int reqoffset, reqlen; /* User-requested offset and length */ | |
340 | unsigned int maxlen; /* Maxmimum permitted length */ | |
341 | ||
342 | size_t sve_state_size; | |
343 | ||
8ae6efdd DM |
344 | const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1, |
345 | SVE_NUM_SLICES - 1); | |
346 | ||
347 | /* Verify that the P-regs and FFR really do have contiguous IDs: */ | |
348 | BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1); | |
349 | ||
350 | /* Verify that we match the UAPI header: */ | |
351 | BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES); | |
352 | ||
e1c9c983 DM |
353 | reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT; |
354 | ||
355 | if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) { | |
52110aa9 DM |
356 | if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) |
357 | return -ENOENT; | |
358 | ||
359 | vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); | |
360 | ||
e1c9c983 DM |
361 | reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - |
362 | SVE_SIG_REGS_OFFSET; | |
363 | reqlen = KVM_SVE_ZREG_SIZE; | |
364 | maxlen = SVE_SIG_ZREG_SIZE(vq); | |
365 | } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) { | |
52110aa9 DM |
366 | if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) |
367 | return -ENOENT; | |
368 | ||
369 | vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); | |
370 | ||
e1c9c983 DM |
371 | reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - |
372 | SVE_SIG_REGS_OFFSET; | |
373 | reqlen = KVM_SVE_PREG_SIZE; | |
374 | maxlen = SVE_SIG_PREG_SIZE(vq); | |
375 | } else { | |
52110aa9 | 376 | return -EINVAL; |
e1c9c983 DM |
377 | } |
378 | ||
379 | sve_state_size = vcpu_sve_state_size(vcpu); | |
55ffad3b | 380 | if (WARN_ON(!sve_state_size)) |
e1c9c983 DM |
381 | return -EINVAL; |
382 | ||
383 | region->koffset = array_index_nospec(reqoffset, sve_state_size); | |
384 | region->klen = min(maxlen, reqlen); | |
385 | region->upad = reqlen - region->klen; | |
386 | ||
387 | return 0; | |
388 | } | |
389 | ||
390 | static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
391 | { | |
52110aa9 | 392 | int ret; |
e1c9c983 DM |
393 | struct sve_state_reg_region region; |
394 | char __user *uptr = (char __user *)reg->addr; | |
395 | ||
9033bba4 DM |
396 | /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ |
397 | if (reg->id == KVM_REG_ARM64_SVE_VLS) | |
398 | return get_sve_vls(vcpu, reg); | |
399 | ||
52110aa9 DM |
400 | /* Try to interpret reg ID as an architectural SVE register... */ |
401 | ret = sve_reg_to_region(®ion, vcpu, reg); | |
402 | if (ret) | |
403 | return ret; | |
9033bba4 DM |
404 | |
405 | if (!kvm_arm_vcpu_sve_finalized(vcpu)) | |
406 | return -EPERM; | |
407 | ||
e1c9c983 DM |
408 | if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, |
409 | region.klen) || | |
410 | clear_user(uptr + region.klen, region.upad)) | |
411 | return -EFAULT; | |
412 | ||
413 | return 0; | |
414 | } | |
415 | ||
416 | static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
417 | { | |
52110aa9 | 418 | int ret; |
e1c9c983 DM |
419 | struct sve_state_reg_region region; |
420 | const char __user *uptr = (const char __user *)reg->addr; | |
421 | ||
9033bba4 DM |
422 | /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ |
423 | if (reg->id == KVM_REG_ARM64_SVE_VLS) | |
424 | return set_sve_vls(vcpu, reg); | |
425 | ||
52110aa9 DM |
426 | /* Try to interpret reg ID as an architectural SVE register... */ |
427 | ret = sve_reg_to_region(®ion, vcpu, reg); | |
428 | if (ret) | |
429 | return ret; | |
9033bba4 DM |
430 | |
431 | if (!kvm_arm_vcpu_sve_finalized(vcpu)) | |
432 | return -EPERM; | |
433 | ||
e1c9c983 DM |
434 | if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, |
435 | region.klen)) | |
436 | return -EFAULT; | |
437 | ||
438 | return 0; | |
439 | } | |
440 | ||
2f4a07c5 MZ |
441 | int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) |
442 | { | |
443 | return -EINVAL; | |
444 | } | |
445 | ||
446 | int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |
447 | { | |
448 | return -EINVAL; | |
449 | } | |
450 | ||
8c86dfe3 DM |
451 | static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, |
452 | u64 __user *uindices) | |
be25bbb3 DM |
453 | { |
454 | unsigned int i; | |
455 | int n = 0; | |
be25bbb3 DM |
456 | |
457 | for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) { | |
df205b5c DM |
458 | u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i; |
459 | int size = core_reg_size_from_offset(vcpu, i); | |
460 | ||
461 | if (size < 0) | |
462 | continue; | |
463 | ||
464 | switch (size) { | |
465 | case sizeof(__u32): | |
466 | reg |= KVM_REG_SIZE_U32; | |
467 | break; | |
468 | ||
469 | case sizeof(__u64): | |
470 | reg |= KVM_REG_SIZE_U64; | |
471 | break; | |
472 | ||
473 | case sizeof(__uint128_t): | |
474 | reg |= KVM_REG_SIZE_U128; | |
475 | break; | |
476 | ||
477 | default: | |
478 | WARN_ON(1); | |
8c86dfe3 | 479 | continue; |
df205b5c | 480 | } |
8c86dfe3 | 481 | |
be25bbb3 | 482 | if (uindices) { |
df205b5c | 483 | if (put_user(reg, uindices)) |
be25bbb3 DM |
484 | return -EFAULT; |
485 | uindices++; | |
486 | } | |
487 | ||
488 | n++; | |
489 | } | |
490 | ||
491 | return n; | |
492 | } | |
493 | ||
8c86dfe3 | 494 | static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) |
2f4a07c5 | 495 | { |
8c86dfe3 | 496 | return copy_core_reg_indices(vcpu, NULL); |
2f4a07c5 MZ |
497 | } |
498 | ||
1df08ba0 AB |
499 | /** |
500 | * ARM64 versions of the TIMER registers, always available on arm64 | |
501 | */ | |
502 | ||
503 | #define NUM_TIMER_REGS 3 | |
504 | ||
505 | static bool is_timer_reg(u64 index) | |
506 | { | |
507 | switch (index) { | |
508 | case KVM_REG_ARM_TIMER_CTL: | |
509 | case KVM_REG_ARM_TIMER_CNT: | |
510 | case KVM_REG_ARM_TIMER_CVAL: | |
511 | return true; | |
512 | } | |
513 | return false; | |
514 | } | |
515 | ||
516 | static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) | |
517 | { | |
518 | if (put_user(KVM_REG_ARM_TIMER_CTL, uindices)) | |
519 | return -EFAULT; | |
520 | uindices++; | |
521 | if (put_user(KVM_REG_ARM_TIMER_CNT, uindices)) | |
522 | return -EFAULT; | |
523 | uindices++; | |
524 | if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices)) | |
525 | return -EFAULT; | |
526 | ||
527 | return 0; | |
528 | } | |
529 | ||
530 | static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
531 | { | |
532 | void __user *uaddr = (void __user *)(long)reg->addr; | |
533 | u64 val; | |
534 | int ret; | |
535 | ||
536 | ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id)); | |
537 | if (ret != 0) | |
bd218bce | 538 | return -EFAULT; |
1df08ba0 AB |
539 | |
540 | return kvm_arm_timer_set_reg(vcpu, reg->id, val); | |
541 | } | |
542 | ||
543 | static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
544 | { | |
545 | void __user *uaddr = (void __user *)(long)reg->addr; | |
546 | u64 val; | |
547 | ||
548 | val = kvm_arm_timer_get_reg(vcpu, reg->id); | |
4cad67fc | 549 | return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0; |
1df08ba0 AB |
550 | } |
551 | ||
8e3c54c8 DM |
552 | static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) |
553 | { | |
8e3c54c8 DM |
554 | const unsigned int slices = vcpu_sve_slices(vcpu); |
555 | ||
556 | if (!vcpu_has_sve(vcpu)) | |
557 | return 0; | |
558 | ||
9033bba4 DM |
559 | /* Policed by KVM_GET_REG_LIST: */ |
560 | WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); | |
561 | ||
562 | return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */) | |
563 | + 1; /* KVM_REG_ARM64_SVE_VLS */ | |
8e3c54c8 DM |
564 | } |
565 | ||
566 | static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu, | |
567 | u64 __user *uindices) | |
568 | { | |
8e3c54c8 DM |
569 | const unsigned int slices = vcpu_sve_slices(vcpu); |
570 | u64 reg; | |
571 | unsigned int i, n; | |
572 | int num_regs = 0; | |
573 | ||
574 | if (!vcpu_has_sve(vcpu)) | |
575 | return 0; | |
576 | ||
9033bba4 DM |
577 | /* Policed by KVM_GET_REG_LIST: */ |
578 | WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); | |
579 | ||
580 | /* | |
581 | * Enumerate this first, so that userspace can save/restore in | |
582 | * the order reported by KVM_GET_REG_LIST: | |
583 | */ | |
584 | reg = KVM_REG_ARM64_SVE_VLS; | |
585 | if (put_user(reg, uindices++)) | |
586 | return -EFAULT; | |
9033bba4 DM |
587 | ++num_regs; |
588 | ||
8e3c54c8 DM |
589 | for (i = 0; i < slices; i++) { |
590 | for (n = 0; n < SVE_NUM_ZREGS; n++) { | |
591 | reg = KVM_REG_ARM64_SVE_ZREG(n, i); | |
592 | if (put_user(reg, uindices++)) | |
593 | return -EFAULT; | |
8e3c54c8 DM |
594 | num_regs++; |
595 | } | |
596 | ||
597 | for (n = 0; n < SVE_NUM_PREGS; n++) { | |
598 | reg = KVM_REG_ARM64_SVE_PREG(n, i); | |
599 | if (put_user(reg, uindices++)) | |
600 | return -EFAULT; | |
8e3c54c8 DM |
601 | num_regs++; |
602 | } | |
603 | ||
604 | reg = KVM_REG_ARM64_SVE_FFR(i); | |
605 | if (put_user(reg, uindices++)) | |
606 | return -EFAULT; | |
8e3c54c8 DM |
607 | num_regs++; |
608 | } | |
609 | ||
610 | return num_regs; | |
611 | } | |
612 | ||
2f4a07c5 MZ |
613 | /** |
614 | * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG | |
615 | * | |
616 | * This is for all registers. | |
617 | */ | |
618 | unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) | |
619 | { | |
7aa92cf3 DM |
620 | unsigned long res = 0; |
621 | ||
8c86dfe3 | 622 | res += num_core_regs(vcpu); |
8e3c54c8 | 623 | res += num_sve_regs(vcpu); |
7aa92cf3 DM |
624 | res += kvm_arm_num_sys_reg_descs(vcpu); |
625 | res += kvm_arm_get_fw_num_regs(vcpu); | |
626 | res += NUM_TIMER_REGS; | |
627 | ||
628 | return res; | |
2f4a07c5 MZ |
629 | } |
630 | ||
631 | /** | |
632 | * kvm_arm_copy_reg_indices - get indices of all registers. | |
633 | * | |
edce2292 | 634 | * We do core registers right here, then we append system regs. |
2f4a07c5 MZ |
635 | */ |
636 | int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) | |
637 | { | |
1df08ba0 | 638 | int ret; |
2f4a07c5 | 639 | |
8c86dfe3 | 640 | ret = copy_core_reg_indices(vcpu, uindices); |
5d8d4af2 | 641 | if (ret < 0) |
be25bbb3 DM |
642 | return ret; |
643 | uindices += ret; | |
2f4a07c5 | 644 | |
8e3c54c8 | 645 | ret = copy_sve_reg_indices(vcpu, uindices); |
5d8d4af2 | 646 | if (ret < 0) |
8e3c54c8 DM |
647 | return ret; |
648 | uindices += ret; | |
649 | ||
85bd0ba1 | 650 | ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); |
5d8d4af2 | 651 | if (ret < 0) |
85bd0ba1 MZ |
652 | return ret; |
653 | uindices += kvm_arm_get_fw_num_regs(vcpu); | |
654 | ||
1df08ba0 | 655 | ret = copy_timer_indices(vcpu, uindices); |
5d8d4af2 | 656 | if (ret < 0) |
1df08ba0 AB |
657 | return ret; |
658 | uindices += NUM_TIMER_REGS; | |
659 | ||
2f4a07c5 MZ |
660 | return kvm_arm_copy_sys_reg_indices(vcpu, uindices); |
661 | } | |
662 | ||
663 | int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
664 | { | |
665 | /* We currently use nothing arch-specific in upper 32 bits */ | |
666 | if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) | |
667 | return -EINVAL; | |
668 | ||
e1c9c983 DM |
669 | switch (reg->id & KVM_REG_ARM_COPROC_MASK) { |
670 | case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); | |
671 | case KVM_REG_ARM_FW: return kvm_arm_get_fw_reg(vcpu, reg); | |
672 | case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); | |
e1c9c983 | 673 | } |
85bd0ba1 | 674 | |
1df08ba0 AB |
675 | if (is_timer_reg(reg->id)) |
676 | return get_timer_reg(vcpu, reg); | |
677 | ||
2f4a07c5 MZ |
678 | return kvm_arm_sys_reg_get_reg(vcpu, reg); |
679 | } | |
680 | ||
681 | int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
682 | { | |
683 | /* We currently use nothing arch-specific in upper 32 bits */ | |
684 | if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) | |
685 | return -EINVAL; | |
686 | ||
e1c9c983 DM |
687 | switch (reg->id & KVM_REG_ARM_COPROC_MASK) { |
688 | case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); | |
689 | case KVM_REG_ARM_FW: return kvm_arm_set_fw_reg(vcpu, reg); | |
690 | case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); | |
e1c9c983 | 691 | } |
85bd0ba1 | 692 | |
1df08ba0 AB |
693 | if (is_timer_reg(reg->id)) |
694 | return set_timer_reg(vcpu, reg); | |
695 | ||
2f4a07c5 MZ |
696 | return kvm_arm_sys_reg_set_reg(vcpu, reg); |
697 | } | |
698 | ||
699 | int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, | |
700 | struct kvm_sregs *sregs) | |
701 | { | |
702 | return -EINVAL; | |
703 | } | |
704 | ||
705 | int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |
706 | struct kvm_sregs *sregs) | |
707 | { | |
708 | return -EINVAL; | |
709 | } | |
710 | ||
539aee0e JM |
711 | int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, |
712 | struct kvm_vcpu_events *events) | |
b7b27fac | 713 | { |
b7b27fac DG |
714 | events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE); |
715 | events->exception.serror_has_esr = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); | |
716 | ||
717 | if (events->exception.serror_pending && events->exception.serror_has_esr) | |
718 | events->exception.serror_esr = vcpu_get_vsesr(vcpu); | |
719 | ||
da345174 CD |
720 | /* |
721 | * We never return a pending ext_dabt here because we deliver it to | |
722 | * the virtual CPU directly when setting the event and it's no longer | |
723 | * 'pending' at this point. | |
724 | */ | |
725 | ||
b7b27fac DG |
726 | return 0; |
727 | } | |
728 | ||
539aee0e JM |
729 | int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, |
730 | struct kvm_vcpu_events *events) | |
b7b27fac | 731 | { |
b7b27fac DG |
732 | bool serror_pending = events->exception.serror_pending; |
733 | bool has_esr = events->exception.serror_has_esr; | |
da345174 | 734 | bool ext_dabt_pending = events->exception.ext_dabt_pending; |
b7b27fac | 735 | |
b7b27fac DG |
736 | if (serror_pending && has_esr) { |
737 | if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) | |
738 | return -EINVAL; | |
739 | ||
740 | if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK)) | |
741 | kvm_set_sei_esr(vcpu, events->exception.serror_esr); | |
742 | else | |
743 | return -EINVAL; | |
744 | } else if (serror_pending) { | |
745 | kvm_inject_vabt(vcpu); | |
746 | } | |
747 | ||
da345174 CD |
748 | if (ext_dabt_pending) |
749 | kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); | |
750 | ||
b7b27fac DG |
751 | return 0; |
752 | } | |
753 | ||
2f4a07c5 MZ |
754 | int __attribute_const__ kvm_target_cpu(void) |
755 | { | |
756 | unsigned long implementor = read_cpuid_implementor(); | |
757 | unsigned long part_number = read_cpuid_part_number(); | |
758 | ||
e28100bd AP |
759 | switch (implementor) { |
760 | case ARM_CPU_IMP_ARM: | |
761 | switch (part_number) { | |
762 | case ARM_CPU_PART_AEM_V8: | |
763 | return KVM_ARM_TARGET_AEM_V8; | |
764 | case ARM_CPU_PART_FOUNDATION: | |
765 | return KVM_ARM_TARGET_FOUNDATION_V8; | |
1252b331 MZ |
766 | case ARM_CPU_PART_CORTEX_A53: |
767 | return KVM_ARM_TARGET_CORTEX_A53; | |
e28100bd AP |
768 | case ARM_CPU_PART_CORTEX_A57: |
769 | return KVM_ARM_TARGET_CORTEX_A57; | |
f0725345 | 770 | } |
e28100bd AP |
771 | break; |
772 | case ARM_CPU_IMP_APM: | |
773 | switch (part_number) { | |
774 | case APM_CPU_PART_POTENZA: | |
775 | return KVM_ARM_TARGET_XGENE_POTENZA; | |
f0725345 | 776 | } |
e28100bd | 777 | break; |
f0725345 | 778 | } |
2f4a07c5 | 779 | |
bca556ac SP |
780 | /* Return a default generic target */ |
781 | return KVM_ARM_TARGET_GENERIC_V8; | |
2f4a07c5 MZ |
782 | } |
783 | ||
473bdc0e AP |
784 | int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) |
785 | { | |
786 | int target = kvm_target_cpu(); | |
787 | ||
788 | if (target < 0) | |
789 | return -ENODEV; | |
790 | ||
791 | memset(init, 0, sizeof(*init)); | |
792 | ||
793 | /* | |
794 | * For now, we don't return any features. | |
795 | * In future, we might use features to return target | |
796 | * specific features available for the preferred | |
797 | * target type. | |
798 | */ | |
799 | init->target = (__u32)target; | |
800 | ||
801 | return 0; | |
802 | } | |
803 | ||
2f4a07c5 MZ |
804 | int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) |
805 | { | |
806 | return -EINVAL; | |
807 | } | |
808 | ||
809 | int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) | |
810 | { | |
811 | return -EINVAL; | |
812 | } | |
813 | ||
814 | int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, | |
815 | struct kvm_translation *tr) | |
816 | { | |
817 | return -EINVAL; | |
818 | } | |
0e6f07f2 | 819 | |
337b99bf AB |
820 | #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ |
821 | KVM_GUESTDBG_USE_SW_BP | \ | |
834bf887 | 822 | KVM_GUESTDBG_USE_HW | \ |
337b99bf | 823 | KVM_GUESTDBG_SINGLESTEP) |
0e6f07f2 AB |
824 | |
825 | /** | |
826 | * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging | |
827 | * @kvm: pointer to the KVM struct | |
828 | * @kvm_guest_debug: the ioctl data buffer | |
829 | * | |
830 | * This sets up and enables the VM for guest debugging. Userspace | |
831 | * passes in a control flag to enable different debug types and | |
832 | * potentially other architecture specific information in the rest of | |
833 | * the structure. | |
834 | */ | |
835 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, | |
836 | struct kvm_guest_debug *dbg) | |
837 | { | |
66b56562 CD |
838 | int ret = 0; |
839 | ||
eef8c85a AB |
840 | trace_kvm_set_guest_debug(vcpu, dbg->control); |
841 | ||
66b56562 CD |
842 | if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) { |
843 | ret = -EINVAL; | |
844 | goto out; | |
845 | } | |
0e6f07f2 AB |
846 | |
847 | if (dbg->control & KVM_GUESTDBG_ENABLE) { | |
848 | vcpu->guest_debug = dbg->control; | |
834bf887 AB |
849 | |
850 | /* Hardware assisted Break and Watch points */ | |
851 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { | |
852 | vcpu->arch.external_debug_state = dbg->arch; | |
853 | } | |
854 | ||
0e6f07f2 AB |
855 | } else { |
856 | /* If not enabled clear all flags */ | |
857 | vcpu->guest_debug = 0; | |
858 | } | |
66b56562 CD |
859 | |
860 | out: | |
66b56562 | 861 | return ret; |
0e6f07f2 | 862 | } |
bb0c70bc SZ |
863 | |
864 | int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, | |
865 | struct kvm_device_attr *attr) | |
866 | { | |
867 | int ret; | |
868 | ||
869 | switch (attr->group) { | |
870 | case KVM_ARM_VCPU_PMU_V3_CTRL: | |
871 | ret = kvm_arm_pmu_v3_set_attr(vcpu, attr); | |
872 | break; | |
99a1db7a CD |
873 | case KVM_ARM_VCPU_TIMER_CTRL: |
874 | ret = kvm_arm_timer_set_attr(vcpu, attr); | |
875 | break; | |
58772e9a SP |
876 | case KVM_ARM_VCPU_PVTIME_CTRL: |
877 | ret = kvm_arm_pvtime_set_attr(vcpu, attr); | |
878 | break; | |
bb0c70bc SZ |
879 | default: |
880 | ret = -ENXIO; | |
881 | break; | |
882 | } | |
883 | ||
884 | return ret; | |
885 | } | |
886 | ||
887 | int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, | |
888 | struct kvm_device_attr *attr) | |
889 | { | |
890 | int ret; | |
891 | ||
892 | switch (attr->group) { | |
893 | case KVM_ARM_VCPU_PMU_V3_CTRL: | |
894 | ret = kvm_arm_pmu_v3_get_attr(vcpu, attr); | |
895 | break; | |
99a1db7a CD |
896 | case KVM_ARM_VCPU_TIMER_CTRL: |
897 | ret = kvm_arm_timer_get_attr(vcpu, attr); | |
898 | break; | |
58772e9a SP |
899 | case KVM_ARM_VCPU_PVTIME_CTRL: |
900 | ret = kvm_arm_pvtime_get_attr(vcpu, attr); | |
901 | break; | |
bb0c70bc SZ |
902 | default: |
903 | ret = -ENXIO; | |
904 | break; | |
905 | } | |
906 | ||
907 | return ret; | |
908 | } | |
909 | ||
910 | int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, | |
911 | struct kvm_device_attr *attr) | |
912 | { | |
913 | int ret; | |
914 | ||
915 | switch (attr->group) { | |
916 | case KVM_ARM_VCPU_PMU_V3_CTRL: | |
917 | ret = kvm_arm_pmu_v3_has_attr(vcpu, attr); | |
918 | break; | |
99a1db7a CD |
919 | case KVM_ARM_VCPU_TIMER_CTRL: |
920 | ret = kvm_arm_timer_has_attr(vcpu, attr); | |
921 | break; | |
58772e9a SP |
922 | case KVM_ARM_VCPU_PVTIME_CTRL: |
923 | ret = kvm_arm_pvtime_has_attr(vcpu, attr); | |
924 | break; | |
bb0c70bc SZ |
925 | default: |
926 | ret = -ENXIO; | |
927 | break; | |
928 | } | |
929 | ||
930 | return ret; | |
931 | } |