]>
Commit | Line | Data |
---|---|---|
1802d0be | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ed9b8cef AP |
2 | /* |
3 | * VGICv3 MMIO handling functions | |
ed9b8cef AP |
4 | */ |
5 | ||
2291ff2f | 6 | #include <linux/bitfield.h> |
ed9b8cef AP |
7 | #include <linux/irqchip/arm-gic-v3.h> |
8 | #include <linux/kvm.h> | |
9 | #include <linux/kvm_host.h> | |
ef1820be | 10 | #include <linux/interrupt.h> |
ed9b8cef AP |
11 | #include <kvm/iodev.h> |
12 | #include <kvm/arm_vgic.h> | |
13 | ||
14 | #include <asm/kvm_emulate.h> | |
94574c94 VK |
15 | #include <asm/kvm_arm.h> |
16 | #include <asm/kvm_mmu.h> | |
ed9b8cef AP |
17 | |
18 | #include "vgic.h" | |
19 | #include "vgic-mmio.h" | |
20 | ||
741972d8 | 21 | /* extract @num bytes at @offset bytes offset in data */ |
d7d0a11e | 22 | unsigned long extract_bytes(u64 data, unsigned int offset, |
424c3383 | 23 | unsigned int num) |
741972d8 AP |
24 | { |
25 | return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0); | |
26 | } | |
27 | ||
0aa1de57 | 28 | /* allows updates of any half of a 64-bit register (or the whole thing) */ |
424c3383 AP |
29 | u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, |
30 | unsigned long val) | |
0aa1de57 AP |
31 | { |
32 | int lower = (offset & 4) * 8; | |
33 | int upper = lower + 8 * len - 1; | |
34 | ||
35 | reg &= ~GENMASK_ULL(upper, lower); | |
36 | val &= GENMASK_ULL(len * 8 - 1, 0); | |
37 | ||
38 | return reg | ((u64)val << lower); | |
39 | } | |
40 | ||
59c5ab40 AP |
41 | bool vgic_has_its(struct kvm *kvm) |
42 | { | |
43 | struct vgic_dist *dist = &kvm->arch.vgic; | |
44 | ||
45 | if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) | |
46 | return false; | |
47 | ||
1085fdc6 | 48 | return dist->has_its; |
59c5ab40 AP |
49 | } |
50 | ||
e7c48059 MZ |
51 | bool vgic_supports_direct_msis(struct kvm *kvm) |
52 | { | |
958e8e14 MZ |
53 | return (kvm_vgic_global_state.has_gicv4_1 || |
54 | (kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm))); | |
e7c48059 MZ |
55 | } |
56 | ||
d53c2c29 CD |
57 | /* |
58 | * The Revision field in the IIDR have the following meanings: | |
59 | * | |
60 | * Revision 2: Interrupt groups are guest-configurable and signaled using | |
61 | * their configured groups. | |
62 | */ | |
63 | ||
fd59ed3b AP |
64 | static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, |
65 | gpa_t addr, unsigned int len) | |
66 | { | |
aa075b0f | 67 | struct vgic_dist *vgic = &vcpu->kvm->arch.vgic; |
fd59ed3b AP |
68 | u32 value = 0; |
69 | ||
70 | switch (addr & 0x0c) { | |
71 | case GICD_CTLR: | |
aa075b0f | 72 | if (vgic->enabled) |
fd59ed3b AP |
73 | value |= GICD_CTLR_ENABLE_SS_G1; |
74 | value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; | |
2291ff2f MZ |
75 | if (vgic->nassgireq) |
76 | value |= GICD_CTLR_nASSGIreq; | |
fd59ed3b AP |
77 | break; |
78 | case GICD_TYPER: | |
aa075b0f | 79 | value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS; |
fd59ed3b | 80 | value = (value >> 5) - 1; |
0e4e82f1 AP |
81 | if (vgic_has_its(vcpu->kvm)) { |
82 | value |= (INTERRUPT_ID_BITS_ITS - 1) << 19; | |
83 | value |= GICD_TYPER_LPIS; | |
84 | } else { | |
85 | value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; | |
86 | } | |
fd59ed3b | 87 | break; |
2291ff2f MZ |
88 | case GICD_TYPER2: |
89 | if (kvm_vgic_global_state.has_gicv4_1) | |
90 | value = GICD_TYPER2_nASSGIcap; | |
91 | break; | |
fd59ed3b | 92 | case GICD_IIDR: |
a2dca217 | 93 | value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) | |
aa075b0f | 94 | (vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) | |
a2dca217 | 95 | (IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT); |
fd59ed3b AP |
96 | break; |
97 | default: | |
98 | return 0; | |
99 | } | |
100 | ||
101 | return value; | |
102 | } | |
103 | ||
104 | static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, | |
105 | gpa_t addr, unsigned int len, | |
106 | unsigned long val) | |
107 | { | |
108 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
fd59ed3b AP |
109 | |
110 | switch (addr & 0x0c) { | |
2291ff2f MZ |
111 | case GICD_CTLR: { |
112 | bool was_enabled, is_hwsgi; | |
113 | ||
114 | mutex_lock(&vcpu->kvm->lock); | |
115 | ||
116 | was_enabled = dist->enabled; | |
117 | is_hwsgi = dist->nassgireq; | |
118 | ||
fd59ed3b AP |
119 | dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; |
120 | ||
2291ff2f MZ |
121 | /* Not a GICv4.1? No HW SGIs */ |
122 | if (!kvm_vgic_global_state.has_gicv4_1) | |
123 | val &= ~GICD_CTLR_nASSGIreq; | |
124 | ||
125 | /* Dist stays enabled? nASSGIreq is RO */ | |
126 | if (was_enabled && dist->enabled) { | |
127 | val &= ~GICD_CTLR_nASSGIreq; | |
128 | val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi); | |
129 | } | |
130 | ||
131 | /* Switching HW SGIs? */ | |
132 | dist->nassgireq = val & GICD_CTLR_nASSGIreq; | |
133 | if (is_hwsgi != dist->nassgireq) | |
134 | vgic_v4_configure_vsgis(vcpu->kvm); | |
135 | ||
d9c3872c MZ |
136 | if (kvm_vgic_global_state.has_gicv4_1 && |
137 | was_enabled != dist->enabled) | |
138 | kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4); | |
139 | else if (!was_enabled && dist->enabled) | |
fd59ed3b | 140 | vgic_kick_vcpus(vcpu->kvm); |
2291ff2f MZ |
141 | |
142 | mutex_unlock(&vcpu->kvm->lock); | |
fd59ed3b | 143 | break; |
2291ff2f | 144 | } |
fd59ed3b | 145 | case GICD_TYPER: |
2291ff2f | 146 | case GICD_TYPER2: |
fd59ed3b | 147 | case GICD_IIDR: |
2291ff2f | 148 | /* This is at best for documentation purposes... */ |
fd59ed3b AP |
149 | return; |
150 | } | |
151 | } | |
152 | ||
b489edc3 CD |
153 | static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, |
154 | gpa_t addr, unsigned int len, | |
155 | unsigned long val) | |
156 | { | |
2291ff2f MZ |
157 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; |
158 | ||
b489edc3 | 159 | switch (addr & 0x0c) { |
2291ff2f | 160 | case GICD_TYPER2: |
b489edc3 CD |
161 | case GICD_IIDR: |
162 | if (val != vgic_mmio_read_v3_misc(vcpu, addr, len)) | |
163 | return -EINVAL; | |
2291ff2f MZ |
164 | return 0; |
165 | case GICD_CTLR: | |
166 | /* Not a GICv4.1? No HW SGIs */ | |
167 | if (!kvm_vgic_global_state.has_gicv4_1) | |
168 | val &= ~GICD_CTLR_nASSGIreq; | |
169 | ||
170 | dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; | |
171 | dist->nassgireq = val & GICD_CTLR_nASSGIreq; | |
172 | return 0; | |
b489edc3 CD |
173 | } |
174 | ||
175 | vgic_mmio_write_v3_misc(vcpu, addr, len, val); | |
176 | return 0; | |
177 | } | |
178 | ||
78a714ab AP |
179 | static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu, |
180 | gpa_t addr, unsigned int len) | |
181 | { | |
182 | int intid = VGIC_ADDR_TO_INTID(addr, 64); | |
183 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid); | |
5dd4b924 | 184 | unsigned long ret = 0; |
78a714ab AP |
185 | |
186 | if (!irq) | |
187 | return 0; | |
188 | ||
189 | /* The upper word is RAZ for us. */ | |
5dd4b924 AP |
190 | if (!(addr & 4)) |
191 | ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len); | |
78a714ab | 192 | |
5dd4b924 AP |
193 | vgic_put_irq(vcpu->kvm, irq); |
194 | return ret; | |
78a714ab AP |
195 | } |
196 | ||
197 | static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, | |
198 | gpa_t addr, unsigned int len, | |
199 | unsigned long val) | |
200 | { | |
201 | int intid = VGIC_ADDR_TO_INTID(addr, 64); | |
5dd4b924 | 202 | struct vgic_irq *irq; |
006df0f3 | 203 | unsigned long flags; |
78a714ab AP |
204 | |
205 | /* The upper word is WI for us since we don't implement Aff3. */ | |
206 | if (addr & 4) | |
207 | return; | |
208 | ||
5dd4b924 AP |
209 | irq = vgic_get_irq(vcpu->kvm, NULL, intid); |
210 | ||
211 | if (!irq) | |
212 | return; | |
213 | ||
8fa3adb8 | 214 | raw_spin_lock_irqsave(&irq->irq_lock, flags); |
78a714ab AP |
215 | |
216 | /* We only care about and preserve Aff0, Aff1 and Aff2. */ | |
217 | irq->mpidr = val & GENMASK(23, 0); | |
218 | irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr); | |
219 | ||
8fa3adb8 | 220 | raw_spin_unlock_irqrestore(&irq->irq_lock, flags); |
5dd4b924 | 221 | vgic_put_irq(vcpu->kvm, irq); |
78a714ab AP |
222 | } |
223 | ||
59c5ab40 AP |
224 | static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, |
225 | gpa_t addr, unsigned int len) | |
226 | { | |
227 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
228 | ||
229 | return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0; | |
230 | } | |
231 | ||
232 | ||
233 | static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, | |
234 | gpa_t addr, unsigned int len, | |
235 | unsigned long val) | |
236 | { | |
237 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
238 | bool was_enabled = vgic_cpu->lpis_enabled; | |
239 | ||
240 | if (!vgic_has_its(vcpu->kvm)) | |
241 | return; | |
242 | ||
243 | vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS; | |
244 | ||
b4931afc | 245 | if (was_enabled && !vgic_cpu->lpis_enabled) { |
96085b94 | 246 | vgic_flush_pending_lpis(vcpu); |
b4931afc MZ |
247 | vgic_its_invalidate_cache(vcpu->kvm); |
248 | } | |
96085b94 | 249 | |
0e4e82f1 AP |
250 | if (!was_enabled && vgic_cpu->lpis_enabled) |
251 | vgic_enable_lpis(vcpu); | |
59c5ab40 AP |
252 | } |
253 | ||
741972d8 AP |
254 | static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, |
255 | gpa_t addr, unsigned int len) | |
256 | { | |
257 | unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu); | |
ba7b3f12 EA |
258 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; |
259 | struct vgic_redist_region *rdreg = vgic_cpu->rdreg; | |
741972d8 | 260 | int target_vcpu_id = vcpu->vcpu_id; |
ba7b3f12 EA |
261 | gpa_t last_rdist_typer = rdreg->base + GICR_TYPER + |
262 | (rdreg->free_index - 1) * KVM_VGIC_V3_REDIST_SIZE; | |
741972d8 AP |
263 | u64 value; |
264 | ||
e533a37f | 265 | value = (u64)(mpidr & GENMASK(23, 0)) << 32; |
741972d8 | 266 | value |= ((target_vcpu_id & 0xffff) << 8); |
ba7b3f12 EA |
267 | |
268 | if (addr == last_rdist_typer) | |
741972d8 | 269 | value |= GICR_TYPER_LAST; |
0e4e82f1 AP |
270 | if (vgic_has_its(vcpu->kvm)) |
271 | value |= GICR_TYPER_PLPIS; | |
741972d8 AP |
272 | |
273 | return extract_bytes(value, addr & 7, len); | |
274 | } | |
275 | ||
276 | static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu, | |
277 | gpa_t addr, unsigned int len) | |
278 | { | |
279 | return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); | |
280 | } | |
281 | ||
54f59d2b AP |
282 | static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu, |
283 | gpa_t addr, unsigned int len) | |
284 | { | |
285 | switch (addr & 0xffff) { | |
286 | case GICD_PIDR2: | |
287 | /* report a GICv3 compliant implementation */ | |
288 | return 0x3b; | |
289 | } | |
290 | ||
291 | return 0; | |
292 | } | |
293 | ||
2df903a8 VK |
294 | static unsigned long vgic_v3_uaccess_read_pending(struct kvm_vcpu *vcpu, |
295 | gpa_t addr, unsigned int len) | |
296 | { | |
297 | u32 intid = VGIC_ADDR_TO_INTID(addr, 1); | |
298 | u32 value = 0; | |
299 | int i; | |
300 | ||
301 | /* | |
302 | * pending state of interrupt is latched in pending_latch variable. | |
303 | * Userspace will save and restore pending state and line_level | |
304 | * separately. | |
2f5947df | 305 | * Refer to Documentation/virt/kvm/devices/arm-vgic-v3.txt |
2df903a8 VK |
306 | * for handling of ISPENDR and ICPENDR. |
307 | */ | |
308 | for (i = 0; i < len * 8; i++) { | |
309 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); | |
ef1820be | 310 | bool state = irq->pending_latch; |
2df903a8 | 311 | |
ef1820be MZ |
312 | if (irq->hw && vgic_irq_is_sgi(irq->intid)) { |
313 | int err; | |
314 | ||
315 | err = irq_get_irqchip_state(irq->host_irq, | |
316 | IRQCHIP_STATE_PENDING, | |
317 | &state); | |
318 | WARN_ON(err); | |
319 | } | |
320 | ||
321 | if (state) | |
2df903a8 VK |
322 | value |= (1U << i); |
323 | ||
324 | vgic_put_irq(vcpu->kvm, irq); | |
325 | } | |
326 | ||
327 | return value; | |
328 | } | |
329 | ||
c6e0917b CD |
330 | static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, |
331 | gpa_t addr, unsigned int len, | |
332 | unsigned long val) | |
2df903a8 VK |
333 | { |
334 | u32 intid = VGIC_ADDR_TO_INTID(addr, 1); | |
335 | int i; | |
006df0f3 | 336 | unsigned long flags; |
2df903a8 VK |
337 | |
338 | for (i = 0; i < len * 8; i++) { | |
339 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); | |
340 | ||
8fa3adb8 | 341 | raw_spin_lock_irqsave(&irq->irq_lock, flags); |
2df903a8 VK |
342 | if (test_bit(i, &val)) { |
343 | /* | |
344 | * pending_latch is set irrespective of irq type | |
345 | * (level or edge) to avoid dependency that VM should | |
346 | * restore irq config before pending info. | |
347 | */ | |
348 | irq->pending_latch = true; | |
006df0f3 | 349 | vgic_queue_irq_unlock(vcpu->kvm, irq, flags); |
2df903a8 VK |
350 | } else { |
351 | irq->pending_latch = false; | |
8fa3adb8 | 352 | raw_spin_unlock_irqrestore(&irq->irq_lock, flags); |
2df903a8 VK |
353 | } |
354 | ||
355 | vgic_put_irq(vcpu->kvm, irq); | |
356 | } | |
c6e0917b CD |
357 | |
358 | return 0; | |
2df903a8 VK |
359 | } |
360 | ||
0aa1de57 AP |
361 | /* We want to avoid outer shareable. */ |
362 | u64 vgic_sanitise_shareability(u64 field) | |
363 | { | |
364 | switch (field) { | |
365 | case GIC_BASER_OuterShareable: | |
366 | return GIC_BASER_InnerShareable; | |
367 | default: | |
368 | return field; | |
369 | } | |
370 | } | |
371 | ||
372 | /* Avoid any inner non-cacheable mapping. */ | |
373 | u64 vgic_sanitise_inner_cacheability(u64 field) | |
374 | { | |
375 | switch (field) { | |
376 | case GIC_BASER_CACHE_nCnB: | |
377 | case GIC_BASER_CACHE_nC: | |
378 | return GIC_BASER_CACHE_RaWb; | |
379 | default: | |
380 | return field; | |
381 | } | |
382 | } | |
383 | ||
384 | /* Non-cacheable or same-as-inner are OK. */ | |
385 | u64 vgic_sanitise_outer_cacheability(u64 field) | |
386 | { | |
387 | switch (field) { | |
388 | case GIC_BASER_CACHE_SameAsInner: | |
389 | case GIC_BASER_CACHE_nC: | |
390 | return field; | |
391 | default: | |
392 | return GIC_BASER_CACHE_nC; | |
393 | } | |
394 | } | |
395 | ||
396 | u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift, | |
397 | u64 (*sanitise_fn)(u64)) | |
398 | { | |
399 | u64 field = (reg & field_mask) >> field_shift; | |
400 | ||
401 | field = sanitise_fn(field) << field_shift; | |
402 | return (reg & ~field_mask) | field; | |
403 | } | |
404 | ||
405 | #define PROPBASER_RES0_MASK \ | |
406 | (GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5)) | |
407 | #define PENDBASER_RES0_MASK \ | |
408 | (BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) | \ | |
409 | GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0)) | |
410 | ||
411 | static u64 vgic_sanitise_pendbaser(u64 reg) | |
412 | { | |
413 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK, | |
414 | GICR_PENDBASER_SHAREABILITY_SHIFT, | |
415 | vgic_sanitise_shareability); | |
416 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK, | |
417 | GICR_PENDBASER_INNER_CACHEABILITY_SHIFT, | |
418 | vgic_sanitise_inner_cacheability); | |
419 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK, | |
420 | GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT, | |
421 | vgic_sanitise_outer_cacheability); | |
422 | ||
423 | reg &= ~PENDBASER_RES0_MASK; | |
0aa1de57 AP |
424 | |
425 | return reg; | |
426 | } | |
427 | ||
428 | static u64 vgic_sanitise_propbaser(u64 reg) | |
429 | { | |
430 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK, | |
431 | GICR_PROPBASER_SHAREABILITY_SHIFT, | |
432 | vgic_sanitise_shareability); | |
433 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK, | |
434 | GICR_PROPBASER_INNER_CACHEABILITY_SHIFT, | |
435 | vgic_sanitise_inner_cacheability); | |
436 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK, | |
437 | GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT, | |
438 | vgic_sanitise_outer_cacheability); | |
439 | ||
440 | reg &= ~PROPBASER_RES0_MASK; | |
0aa1de57 AP |
441 | return reg; |
442 | } | |
443 | ||
444 | static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu, | |
445 | gpa_t addr, unsigned int len) | |
446 | { | |
447 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
448 | ||
449 | return extract_bytes(dist->propbaser, addr & 7, len); | |
450 | } | |
451 | ||
452 | static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu, | |
453 | gpa_t addr, unsigned int len, | |
454 | unsigned long val) | |
455 | { | |
456 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
457 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
d9ae449b | 458 | u64 old_propbaser, propbaser; |
0aa1de57 AP |
459 | |
460 | /* Storing a value with LPIs already enabled is undefined */ | |
461 | if (vgic_cpu->lpis_enabled) | |
462 | return; | |
463 | ||
d9ae449b | 464 | do { |
3af4e414 | 465 | old_propbaser = READ_ONCE(dist->propbaser); |
d9ae449b CD |
466 | propbaser = old_propbaser; |
467 | propbaser = update_64bit_reg(propbaser, addr & 4, len, val); | |
468 | propbaser = vgic_sanitise_propbaser(propbaser); | |
469 | } while (cmpxchg64(&dist->propbaser, old_propbaser, | |
470 | propbaser) != old_propbaser); | |
0aa1de57 AP |
471 | } |
472 | ||
473 | static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu, | |
474 | gpa_t addr, unsigned int len) | |
475 | { | |
476 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
5f675c56 | 477 | u64 value = vgic_cpu->pendbaser; |
0aa1de57 | 478 | |
5f675c56 ZY |
479 | value &= ~GICR_PENDBASER_PTZ; |
480 | ||
481 | return extract_bytes(value, addr & 7, len); | |
0aa1de57 AP |
482 | } |
483 | ||
484 | static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, | |
485 | gpa_t addr, unsigned int len, | |
486 | unsigned long val) | |
487 | { | |
488 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
d9ae449b | 489 | u64 old_pendbaser, pendbaser; |
0aa1de57 AP |
490 | |
491 | /* Storing a value with LPIs already enabled is undefined */ | |
492 | if (vgic_cpu->lpis_enabled) | |
493 | return; | |
494 | ||
d9ae449b | 495 | do { |
3af4e414 | 496 | old_pendbaser = READ_ONCE(vgic_cpu->pendbaser); |
d9ae449b CD |
497 | pendbaser = old_pendbaser; |
498 | pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val); | |
499 | pendbaser = vgic_sanitise_pendbaser(pendbaser); | |
500 | } while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser, | |
501 | pendbaser) != old_pendbaser); | |
0aa1de57 AP |
502 | } |
503 | ||
ed9b8cef AP |
504 | /* |
505 | * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the | |
506 | * redistributors, while SPIs are covered by registers in the distributor | |
507 | * block. Trying to set private IRQs in this block gets ignored. | |
508 | * We take some special care here to fix the calculation of the register | |
509 | * offset. | |
510 | */ | |
2df903a8 | 511 | #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \ |
ed9b8cef AP |
512 | { \ |
513 | .reg_offset = off, \ | |
514 | .bits_per_irq = bpi, \ | |
515 | .len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ | |
516 | .access_flags = acc, \ | |
517 | .read = vgic_mmio_read_raz, \ | |
518 | .write = vgic_mmio_write_wi, \ | |
519 | }, { \ | |
520 | .reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ | |
521 | .bits_per_irq = bpi, \ | |
522 | .len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8, \ | |
523 | .access_flags = acc, \ | |
524 | .read = rd, \ | |
525 | .write = wr, \ | |
2df903a8 VK |
526 | .uaccess_read = ur, \ |
527 | .uaccess_write = uw, \ | |
ed9b8cef AP |
528 | } |
529 | ||
530 | static const struct vgic_register_region vgic_v3_dist_registers[] = { | |
b489edc3 CD |
531 | REGISTER_DESC_WITH_LENGTH_UACCESS(GICD_CTLR, |
532 | vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc, | |
533 | NULL, vgic_mmio_uaccess_write_v3_misc, | |
534 | 16, VGIC_ACCESS_32bit), | |
94574c94 VK |
535 | REGISTER_DESC_WITH_LENGTH(GICD_STATUSR, |
536 | vgic_mmio_read_rao, vgic_mmio_write_wi, 4, | |
537 | VGIC_ACCESS_32bit), | |
ed9b8cef | 538 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR, |
d53c2c29 | 539 | vgic_mmio_read_group, vgic_mmio_write_group, NULL, NULL, 1, |
ed9b8cef AP |
540 | VGIC_ACCESS_32bit), |
541 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER, | |
41ee52ec MZ |
542 | vgic_mmio_read_enable, vgic_mmio_write_senable, |
543 | NULL, vgic_uaccess_write_senable, 1, | |
ed9b8cef AP |
544 | VGIC_ACCESS_32bit), |
545 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER, | |
41ee52ec MZ |
546 | vgic_mmio_read_enable, vgic_mmio_write_cenable, |
547 | NULL, vgic_uaccess_write_cenable, 1, | |
ed9b8cef AP |
548 | VGIC_ACCESS_32bit), |
549 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR, | |
2df903a8 VK |
550 | vgic_mmio_read_pending, vgic_mmio_write_spending, |
551 | vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1, | |
ed9b8cef AP |
552 | VGIC_ACCESS_32bit), |
553 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR, | |
2df903a8 | 554 | vgic_mmio_read_pending, vgic_mmio_write_cpending, |
c6e0917b | 555 | vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 1, |
ed9b8cef AP |
556 | VGIC_ACCESS_32bit), |
557 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER, | |
3197191e | 558 | vgic_mmio_read_active, vgic_mmio_write_sactive, |
9a50ebbf | 559 | vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1, |
ed9b8cef AP |
560 | VGIC_ACCESS_32bit), |
561 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER, | |
3197191e | 562 | vgic_mmio_read_active, vgic_mmio_write_cactive, |
9a50ebbf | 563 | vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, |
3197191e | 564 | 1, VGIC_ACCESS_32bit), |
ed9b8cef | 565 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR, |
2df903a8 VK |
566 | vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL, |
567 | 8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
ed9b8cef | 568 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR, |
2df903a8 | 569 | vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8, |
ed9b8cef AP |
570 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), |
571 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR, | |
2df903a8 | 572 | vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2, |
ed9b8cef AP |
573 | VGIC_ACCESS_32bit), |
574 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR, | |
2df903a8 | 575 | vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1, |
ed9b8cef AP |
576 | VGIC_ACCESS_32bit), |
577 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER, | |
2df903a8 | 578 | vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64, |
ed9b8cef AP |
579 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
580 | REGISTER_DESC_WITH_LENGTH(GICD_IDREGS, | |
54f59d2b | 581 | vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, |
ed9b8cef AP |
582 | VGIC_ACCESS_32bit), |
583 | }; | |
584 | ||
3109741a EA |
585 | static const struct vgic_register_region vgic_v3_rd_registers[] = { |
586 | /* RD_base registers */ | |
ed9b8cef | 587 | REGISTER_DESC_WITH_LENGTH(GICR_CTLR, |
59c5ab40 | 588 | vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4, |
ed9b8cef | 589 | VGIC_ACCESS_32bit), |
94574c94 VK |
590 | REGISTER_DESC_WITH_LENGTH(GICR_STATUSR, |
591 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
592 | VGIC_ACCESS_32bit), | |
ed9b8cef | 593 | REGISTER_DESC_WITH_LENGTH(GICR_IIDR, |
741972d8 | 594 | vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4, |
ed9b8cef AP |
595 | VGIC_ACCESS_32bit), |
596 | REGISTER_DESC_WITH_LENGTH(GICR_TYPER, | |
741972d8 | 597 | vgic_mmio_read_v3r_typer, vgic_mmio_write_wi, 8, |
ed9b8cef | 598 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
94574c94 VK |
599 | REGISTER_DESC_WITH_LENGTH(GICR_WAKER, |
600 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
601 | VGIC_ACCESS_32bit), | |
ed9b8cef | 602 | REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER, |
0aa1de57 | 603 | vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8, |
ed9b8cef AP |
604 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
605 | REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER, | |
0aa1de57 | 606 | vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8, |
ed9b8cef AP |
607 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
608 | REGISTER_DESC_WITH_LENGTH(GICR_IDREGS, | |
54f59d2b | 609 | vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, |
ed9b8cef | 610 | VGIC_ACCESS_32bit), |
3109741a EA |
611 | /* SGI_base registers */ |
612 | REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0, | |
d53c2c29 | 613 | vgic_mmio_read_group, vgic_mmio_write_group, 4, |
ed9b8cef | 614 | VGIC_ACCESS_32bit), |
41ee52ec MZ |
615 | REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISENABLER0, |
616 | vgic_mmio_read_enable, vgic_mmio_write_senable, | |
617 | NULL, vgic_uaccess_write_senable, 4, | |
ed9b8cef | 618 | VGIC_ACCESS_32bit), |
41ee52ec MZ |
619 | REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICENABLER0, |
620 | vgic_mmio_read_enable, vgic_mmio_write_cenable, | |
621 | NULL, vgic_uaccess_write_cenable, 4, | |
ed9b8cef | 622 | VGIC_ACCESS_32bit), |
3109741a | 623 | REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0, |
2df903a8 VK |
624 | vgic_mmio_read_pending, vgic_mmio_write_spending, |
625 | vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4, | |
ed9b8cef | 626 | VGIC_ACCESS_32bit), |
3109741a | 627 | REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0, |
2df903a8 | 628 | vgic_mmio_read_pending, vgic_mmio_write_cpending, |
c6e0917b | 629 | vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4, |
ed9b8cef | 630 | VGIC_ACCESS_32bit), |
3109741a | 631 | REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0, |
0710f9a6 | 632 | vgic_mmio_read_active, vgic_mmio_write_sactive, |
9a50ebbf MZ |
633 | vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 4, |
634 | VGIC_ACCESS_32bit), | |
3109741a | 635 | REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0, |
0710f9a6 | 636 | vgic_mmio_read_active, vgic_mmio_write_cactive, |
9a50ebbf MZ |
637 | vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 4, |
638 | VGIC_ACCESS_32bit), | |
3109741a | 639 | REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0, |
ed9b8cef AP |
640 | vgic_mmio_read_priority, vgic_mmio_write_priority, 32, |
641 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
3109741a | 642 | REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0, |
ed9b8cef AP |
643 | vgic_mmio_read_config, vgic_mmio_write_config, 8, |
644 | VGIC_ACCESS_32bit), | |
3109741a | 645 | REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0, |
ed9b8cef AP |
646 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, |
647 | VGIC_ACCESS_32bit), | |
3109741a | 648 | REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR, |
ed9b8cef AP |
649 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, |
650 | VGIC_ACCESS_32bit), | |
651 | }; | |
652 | ||
653 | unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev) | |
654 | { | |
655 | dev->regions = vgic_v3_dist_registers; | |
656 | dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); | |
657 | ||
658 | kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); | |
659 | ||
660 | return SZ_64K; | |
661 | } | |
662 | ||
7fadcd3a CD |
663 | /** |
664 | * vgic_register_redist_iodev - register a single redist iodev | |
665 | * @vcpu: The VCPU to which the redistributor belongs | |
666 | * | |
667 | * Register a KVM iodev for this VCPU's redistributor using the address | |
668 | * provided. | |
669 | * | |
670 | * Return 0 on success, -ERRNO otherwise. | |
671 | */ | |
1aab6f46 | 672 | int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) |
7fadcd3a CD |
673 | { |
674 | struct kvm *kvm = vcpu->kvm; | |
675 | struct vgic_dist *vgic = &kvm->arch.vgic; | |
dbd9733a | 676 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; |
7fadcd3a | 677 | struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; |
dbd9733a | 678 | struct vgic_redist_region *rdreg; |
3109741a | 679 | gpa_t rd_base; |
7fadcd3a CD |
680 | int ret; |
681 | ||
c011f4ea EA |
682 | if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr)) |
683 | return 0; | |
684 | ||
1aab6f46 CD |
685 | /* |
686 | * We may be creating VCPUs before having set the base address for the | |
687 | * redistributor region, in which case we will come back to this | |
688 | * function for all VCPUs when the base address is set. Just return | |
689 | * without doing any work for now. | |
690 | */ | |
dc524619 | 691 | rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions); |
dbd9733a | 692 | if (!rdreg) |
1aab6f46 CD |
693 | return 0; |
694 | ||
695 | if (!vgic_v3_check_base(kvm)) | |
696 | return -EINVAL; | |
697 | ||
dbd9733a EA |
698 | vgic_cpu->rdreg = rdreg; |
699 | ||
700 | rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE; | |
7fadcd3a CD |
701 | |
702 | kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops); | |
703 | rd_dev->base_addr = rd_base; | |
704 | rd_dev->iodev_type = IODEV_REDIST; | |
3109741a EA |
705 | rd_dev->regions = vgic_v3_rd_registers; |
706 | rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); | |
7fadcd3a CD |
707 | rd_dev->redist_vcpu = vcpu; |
708 | ||
709 | mutex_lock(&kvm->slots_lock); | |
710 | ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, | |
3109741a | 711 | 2 * SZ_64K, &rd_dev->dev); |
7fadcd3a CD |
712 | mutex_unlock(&kvm->slots_lock); |
713 | ||
714 | if (ret) | |
715 | return ret; | |
716 | ||
dbd9733a | 717 | rdreg->free_index++; |
3109741a | 718 | return 0; |
7fadcd3a CD |
719 | } |
720 | ||
721 | static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) | |
722 | { | |
723 | struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; | |
7fadcd3a CD |
724 | |
725 | kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev); | |
7fadcd3a CD |
726 | } |
727 | ||
1aab6f46 | 728 | static int vgic_register_all_redist_iodevs(struct kvm *kvm) |
ed9b8cef | 729 | { |
ed9b8cef | 730 | struct kvm_vcpu *vcpu; |
ed9b8cef AP |
731 | int c, ret = 0; |
732 | ||
ed9b8cef | 733 | kvm_for_each_vcpu(c, vcpu, kvm) { |
7fadcd3a | 734 | ret = vgic_register_redist_iodev(vcpu); |
ed9b8cef AP |
735 | if (ret) |
736 | break; | |
ed9b8cef AP |
737 | } |
738 | ||
739 | if (ret) { | |
740 | /* The current c failed, so we start with the previous one. */ | |
fa472fa9 | 741 | mutex_lock(&kvm->slots_lock); |
ed9b8cef | 742 | for (c--; c >= 0; c--) { |
8f6cdc1c | 743 | vcpu = kvm_get_vcpu(kvm, c); |
7fadcd3a | 744 | vgic_unregister_redist_iodev(vcpu); |
ed9b8cef | 745 | } |
fa472fa9 | 746 | mutex_unlock(&kvm->slots_lock); |
ed9b8cef AP |
747 | } |
748 | ||
749 | return ret; | |
750 | } | |
621ecd8d | 751 | |
ccc27bf5 EA |
752 | /** |
753 | * vgic_v3_insert_redist_region - Insert a new redistributor region | |
754 | * | |
755 | * Performs various checks before inserting the rdist region in the list. | |
756 | * Those tests depend on whether the size of the rdist region is known | |
757 | * (ie. count != 0). The list is sorted by rdist region index. | |
758 | * | |
759 | * @kvm: kvm handle | |
760 | * @index: redist region index | |
761 | * @base: base of the new rdist region | |
762 | * @count: number of redistributors the region is made of (0 in the old style | |
763 | * single region, whose size is induced from the number of vcpus) | |
764 | * | |
765 | * Return 0 on success, < 0 otherwise | |
766 | */ | |
767 | static int vgic_v3_insert_redist_region(struct kvm *kvm, uint32_t index, | |
768 | gpa_t base, uint32_t count) | |
1aab6f46 | 769 | { |
ccc27bf5 | 770 | struct vgic_dist *d = &kvm->arch.vgic; |
dbd9733a | 771 | struct vgic_redist_region *rdreg; |
ccc27bf5 EA |
772 | struct list_head *rd_regions = &d->rd_regions; |
773 | size_t size = count * KVM_VGIC_V3_REDIST_SIZE; | |
1aab6f46 CD |
774 | int ret; |
775 | ||
ccc27bf5 EA |
776 | /* single rdist region already set ?*/ |
777 | if (!count && !list_empty(rd_regions)) | |
778 | return -EINVAL; | |
779 | ||
780 | /* cross the end of memory ? */ | |
781 | if (base + size < base) | |
782 | return -EINVAL; | |
783 | ||
784 | if (list_empty(rd_regions)) { | |
785 | if (index != 0) | |
786 | return -EINVAL; | |
787 | } else { | |
788 | rdreg = list_last_entry(rd_regions, | |
789 | struct vgic_redist_region, list); | |
790 | if (index != rdreg->index + 1) | |
791 | return -EINVAL; | |
792 | ||
793 | /* Cannot add an explicitly sized regions after legacy region */ | |
794 | if (!rdreg->count) | |
795 | return -EINVAL; | |
796 | } | |
797 | ||
798 | /* | |
799 | * For legacy single-region redistributor regions (!count), | |
800 | * check that the redistributor region does not overlap with the | |
801 | * distributor's address space. | |
802 | */ | |
803 | if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) && | |
804 | vgic_dist_overlap(kvm, base, size)) | |
805 | return -EINVAL; | |
806 | ||
807 | /* collision with any other rdist region? */ | |
808 | if (vgic_v3_rdist_overlap(kvm, base, size)) | |
dbd9733a EA |
809 | return -EINVAL; |
810 | ||
811 | rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL); | |
812 | if (!rdreg) | |
813 | return -ENOMEM; | |
814 | ||
815 | rdreg->base = VGIC_ADDR_UNDEF; | |
816 | ||
ccc27bf5 | 817 | ret = vgic_check_ioaddr(kvm, &rdreg->base, base, SZ_64K); |
1aab6f46 | 818 | if (ret) |
ccc27bf5 | 819 | goto free; |
1aab6f46 | 820 | |
ccc27bf5 EA |
821 | rdreg->base = base; |
822 | rdreg->count = count; | |
823 | rdreg->free_index = 0; | |
824 | rdreg->index = index; | |
1aab6f46 | 825 | |
ccc27bf5 EA |
826 | list_add_tail(&rdreg->list, rd_regions); |
827 | return 0; | |
828 | free: | |
829 | kfree(rdreg); | |
830 | return ret; | |
831 | } | |
832 | ||
04c11093 | 833 | int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) |
ccc27bf5 EA |
834 | { |
835 | int ret; | |
836 | ||
04c11093 | 837 | ret = vgic_v3_insert_redist_region(kvm, index, addr, count); |
ccc27bf5 EA |
838 | if (ret) |
839 | return ret; | |
dbd9733a | 840 | |
1aab6f46 CD |
841 | /* |
842 | * Register iodevs for each existing VCPU. Adding more VCPUs | |
843 | * afterwards will register the iodevs when needed. | |
844 | */ | |
845 | ret = vgic_register_all_redist_iodevs(kvm); | |
846 | if (ret) | |
847 | return ret; | |
848 | ||
849 | return 0; | |
850 | } | |
851 | ||
94574c94 VK |
852 | int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) |
853 | { | |
854 | const struct vgic_register_region *region; | |
855 | struct vgic_io_device iodev; | |
856 | struct vgic_reg_attr reg_attr; | |
857 | struct kvm_vcpu *vcpu; | |
858 | gpa_t addr; | |
859 | int ret; | |
860 | ||
861 | ret = vgic_v3_parse_attr(dev, attr, ®_attr); | |
862 | if (ret) | |
863 | return ret; | |
864 | ||
865 | vcpu = reg_attr.vcpu; | |
866 | addr = reg_attr.addr; | |
867 | ||
868 | switch (attr->group) { | |
869 | case KVM_DEV_ARM_VGIC_GRP_DIST_REGS: | |
870 | iodev.regions = vgic_v3_dist_registers; | |
871 | iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); | |
872 | iodev.base_addr = 0; | |
873 | break; | |
874 | case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{ | |
3109741a EA |
875 | iodev.regions = vgic_v3_rd_registers; |
876 | iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); | |
94574c94 VK |
877 | iodev.base_addr = 0; |
878 | break; | |
879 | } | |
d017d7b0 VK |
880 | case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: { |
881 | u64 reg, id; | |
882 | ||
883 | id = (attr->attr & KVM_DEV_ARM_VGIC_SYSREG_INSTR_MASK); | |
884 | return vgic_v3_has_cpu_sysregs_attr(vcpu, 0, id, ®); | |
885 | } | |
94574c94 VK |
886 | default: |
887 | return -ENXIO; | |
888 | } | |
889 | ||
890 | /* We only support aligned 32-bit accesses. */ | |
891 | if (addr & 3) | |
892 | return -ENXIO; | |
893 | ||
894 | region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32)); | |
895 | if (!region) | |
896 | return -ENXIO; | |
897 | ||
898 | return 0; | |
899 | } | |
621ecd8d AP |
900 | /* |
901 | * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI | |
902 | * generation register ICC_SGI1R_EL1) with a given VCPU. | |
903 | * If the VCPU's MPIDR matches, return the level0 affinity, otherwise | |
904 | * return -1. | |
905 | */ | |
906 | static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu) | |
907 | { | |
908 | unsigned long affinity; | |
909 | int level0; | |
910 | ||
911 | /* | |
912 | * Split the current VCPU's MPIDR into affinity level 0 and the | |
913 | * rest as this is what we have to compare against. | |
914 | */ | |
915 | affinity = kvm_vcpu_get_mpidr_aff(vcpu); | |
916 | level0 = MPIDR_AFFINITY_LEVEL(affinity, 0); | |
917 | affinity &= ~MPIDR_LEVEL_MASK; | |
918 | ||
919 | /* bail out if the upper three levels don't match */ | |
920 | if (sgi_aff != affinity) | |
921 | return -1; | |
922 | ||
923 | /* Is this VCPU's bit set in the mask ? */ | |
924 | if (!(sgi_cpu_mask & BIT(level0))) | |
925 | return -1; | |
926 | ||
927 | return level0; | |
928 | } | |
929 | ||
930 | /* | |
931 | * The ICC_SGI* registers encode the affinity differently from the MPIDR, | |
932 | * so provide a wrapper to use the existing defines to isolate a certain | |
933 | * affinity level. | |
934 | */ | |
935 | #define SGI_AFFINITY_LEVEL(reg, level) \ | |
936 | ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \ | |
937 | >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level)) | |
938 | ||
939 | /** | |
940 | * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs | |
941 | * @vcpu: The VCPU requesting a SGI | |
6249f2a4 MZ |
942 | * @reg: The value written into ICC_{ASGI1,SGI0,SGI1}R by that VCPU |
943 | * @allow_group1: Does the sysreg access allow generation of G1 SGIs | |
621ecd8d AP |
944 | * |
945 | * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register. | |
946 | * This will trap in sys_regs.c and call this function. | |
947 | * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the | |
948 | * target processors as well as a bitmask of 16 Aff0 CPUs. | |
949 | * If the interrupt routing mode bit is not set, we iterate over all VCPUs to | |
950 | * check for matching ones. If this bit is set, we signal all, but not the | |
951 | * calling VCPU. | |
952 | */ | |
6249f2a4 | 953 | void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1) |
621ecd8d AP |
954 | { |
955 | struct kvm *kvm = vcpu->kvm; | |
956 | struct kvm_vcpu *c_vcpu; | |
957 | u16 target_cpus; | |
958 | u64 mpidr; | |
959 | int sgi, c; | |
960 | int vcpu_id = vcpu->vcpu_id; | |
961 | bool broadcast; | |
006df0f3 | 962 | unsigned long flags; |
621ecd8d AP |
963 | |
964 | sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; | |
e533a37f | 965 | broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); |
621ecd8d AP |
966 | target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT; |
967 | mpidr = SGI_AFFINITY_LEVEL(reg, 3); | |
968 | mpidr |= SGI_AFFINITY_LEVEL(reg, 2); | |
969 | mpidr |= SGI_AFFINITY_LEVEL(reg, 1); | |
970 | ||
971 | /* | |
972 | * We iterate over all VCPUs to find the MPIDRs matching the request. | |
973 | * If we have handled one CPU, we clear its bit to detect early | |
974 | * if we are already finished. This avoids iterating through all | |
975 | * VCPUs when most of the times we just signal a single VCPU. | |
976 | */ | |
977 | kvm_for_each_vcpu(c, c_vcpu, kvm) { | |
978 | struct vgic_irq *irq; | |
979 | ||
980 | /* Exit early if we have dealt with all requested CPUs */ | |
981 | if (!broadcast && target_cpus == 0) | |
982 | break; | |
983 | ||
984 | /* Don't signal the calling VCPU */ | |
985 | if (broadcast && c == vcpu_id) | |
986 | continue; | |
987 | ||
988 | if (!broadcast) { | |
989 | int level0; | |
990 | ||
991 | level0 = match_mpidr(mpidr, target_cpus, c_vcpu); | |
992 | if (level0 == -1) | |
993 | continue; | |
994 | ||
995 | /* remove this matching VCPU from the mask */ | |
996 | target_cpus &= ~BIT(level0); | |
997 | } | |
998 | ||
999 | irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi); | |
1000 | ||
8fa3adb8 | 1001 | raw_spin_lock_irqsave(&irq->irq_lock, flags); |
621ecd8d | 1002 | |
6249f2a4 MZ |
1003 | /* |
1004 | * An access targetting Group0 SGIs can only generate | |
1005 | * those, while an access targetting Group1 SGIs can | |
1006 | * generate interrupts of either group. | |
1007 | */ | |
1008 | if (!irq->group || allow_group1) { | |
ef1820be MZ |
1009 | if (!irq->hw) { |
1010 | irq->pending_latch = true; | |
1011 | vgic_queue_irq_unlock(vcpu->kvm, irq, flags); | |
1012 | } else { | |
1013 | /* HW SGI? Ask the GIC to inject it */ | |
1014 | int err; | |
1015 | err = irq_set_irqchip_state(irq->host_irq, | |
1016 | IRQCHIP_STATE_PENDING, | |
1017 | true); | |
1018 | WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); | |
1019 | raw_spin_unlock_irqrestore(&irq->irq_lock, flags); | |
1020 | } | |
6249f2a4 | 1021 | } else { |
8fa3adb8 | 1022 | raw_spin_unlock_irqrestore(&irq->irq_lock, flags); |
6249f2a4 MZ |
1023 | } |
1024 | ||
5dd4b924 | 1025 | vgic_put_irq(vcpu->kvm, irq); |
621ecd8d AP |
1026 | } |
1027 | } | |
94574c94 VK |
1028 | |
1029 | int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write, | |
1030 | int offset, u32 *val) | |
1031 | { | |
1032 | struct vgic_io_device dev = { | |
1033 | .regions = vgic_v3_dist_registers, | |
1034 | .nr_regions = ARRAY_SIZE(vgic_v3_dist_registers), | |
1035 | }; | |
1036 | ||
1037 | return vgic_uaccess(vcpu, &dev, is_write, offset, val); | |
1038 | } | |
1039 | ||
1040 | int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write, | |
1041 | int offset, u32 *val) | |
1042 | { | |
1043 | struct vgic_io_device rd_dev = { | |
3109741a EA |
1044 | .regions = vgic_v3_rd_registers, |
1045 | .nr_regions = ARRAY_SIZE(vgic_v3_rd_registers), | |
94574c94 VK |
1046 | }; |
1047 | ||
3109741a | 1048 | return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val); |
94574c94 | 1049 | } |
e96a006c VK |
1050 | |
1051 | int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write, | |
1052 | u32 intid, u64 *val) | |
1053 | { | |
1054 | if (intid % 32) | |
1055 | return -EINVAL; | |
1056 | ||
1057 | if (is_write) | |
1058 | vgic_write_irq_line_level_info(vcpu, intid, *val); | |
1059 | else | |
1060 | *val = vgic_read_irq_line_level_info(vcpu, intid); | |
1061 | ||
1062 | return 0; | |
1063 | } |