]> git.ipfire.org Git - thirdparty/linux.git/blame - arch/s390/kvm/kvm-s390.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block
[thirdparty/linux.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
d809aa23 1// SPDX-License-Identifier: GPL-2.0
b0c632db 2/*
bb64da9a 3 * hosting IBM Z kernel virtual machines (s390x)
b0c632db 4 *
3e6c5568 5 * Copyright IBM Corp. 2008, 2020
b0c632db
HC		 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 11 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 12 */
13
7aedd9d4
MM		 14#define KMSG_COMPONENT "kvm-s390"
15#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
16
b0c632db
HC		 17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
d3217967 26#include <linux/moduleparam.h>
a374e892 27#include <linux/random.h>
b0c632db 28#include <linux/slab.h>
ba5c1e9b 29#include <linux/timer.h>
41408c28 30#include <linux/vmalloc.h>
15c9705f 31#include <linux/bitmap.h>
174cd4b1 32#include <linux/sched/signal.h>
190df4a2 33#include <linux/string.h>
174cd4b1 34
cbb870c8 35#include <asm/asm-offsets.h>
b0c632db 36#include <asm/lowcore.h>
fd5ada04 37#include <asm/stp.h>
b0c632db 38#include <asm/pgtable.h>
1e133ab2 39#include <asm/gmap.h>
f5daba1d 40#include <asm/nmi.h>
a0616cde 41#include <asm/switch_to.h>
6d3da241 42#include <asm/isc.h>
1526bf9c 43#include <asm/sclp.h>
0a763c78 44#include <asm/cpacf.h>
221bb8a4 45#include <asm/timex.h>
e585b24a 46#include <asm/ap.h>
29b40f10 47#include <asm/uv.h>
8f2abe6a 48#include "kvm-s390.h"
b0c632db
HC		 49#include "gaccess.h"
50
5786fffa
CH		 51#define CREATE_TRACE_POINTS
52#include "trace.h"
ade38c31 53#include "trace-s390.h"
5786fffa 54
41408c28 55#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 56#define LOCAL_IRQS 32
57#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
58 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 59
b0c632db 60#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
ccc40c53 61#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
b0c632db
HC		 62
63struct kvm_stats_debugfs_item debugfs_entries[] = {
64 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 65 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 66 { "exit_validity", VCPU_STAT(exit_validity) },
67 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
68 { "exit_external_request", VCPU_STAT(exit_external_request) },
a5e0acea 69 { "exit_io_request", VCPU_STAT(exit_io_request) },
8f2abe6a 70 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 71 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 72 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 73 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
74 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 75 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 76 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 77 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 78 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
8b905d28 79 { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
ce2e4f0b 80 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 81 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 82 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 83 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
84 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ccc40c53
CB		 85 { "deliver_ckc", VCPU_STAT(deliver_ckc) },
86 { "deliver_cputm", VCPU_STAT(deliver_cputm) },
ba5c1e9b 87 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 88 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b 89 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
ccc40c53 90 { "deliver_virtio", VCPU_STAT(deliver_virtio) },
ba5c1e9b
CO		 91 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
92 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
93 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
ccc40c53
CB		 94 { "deliver_program", VCPU_STAT(deliver_program) },
95 { "deliver_io", VCPU_STAT(deliver_io) },
32de0749 96 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
ba5c1e9b 97 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
ccc40c53
CB		 98 { "inject_ckc", VCPU_STAT(inject_ckc) },
99 { "inject_cputm", VCPU_STAT(inject_cputm) },
100 { "inject_external_call", VCPU_STAT(inject_external_call) },
101 { "inject_float_mchk", VM_STAT(inject_float_mchk) },
102 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
103 { "inject_io", VM_STAT(inject_io) },
104 { "inject_mchk", VCPU_STAT(inject_mchk) },
105 { "inject_pfault_done", VM_STAT(inject_pfault_done) },
106 { "inject_program", VCPU_STAT(inject_program) },
107 { "inject_restart", VCPU_STAT(inject_restart) },
108 { "inject_service_signal", VM_STAT(inject_service_signal) },
109 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
110 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
111 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
112 { "inject_virtio", VM_STAT(inject_virtio) },
a37cb07a
CB		 113 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
114 { "instruction_gs", VCPU_STAT(instruction_gs) },
115 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
116 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
117 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
69d0d3a3 118 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
a37cb07a 119 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
453423dc 120 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
a37cb07a
CB		 121 { "instruction_sck", VCPU_STAT(instruction_sck) },
122 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
453423dc
CB		 123 { "instruction_spx", VCPU_STAT(instruction_spx) },
124 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
125 { "instruction_stap", VCPU_STAT(instruction_stap) },
a37cb07a
CB		 126 { "instruction_iske", VCPU_STAT(instruction_iske) },
127 { "instruction_ri", VCPU_STAT(instruction_ri) },
128 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
129 { "instruction_sske", VCPU_STAT(instruction_sske) },
8a242234 130 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
b31288fa 131 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 132 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
133 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
a37cb07a
CB		 134 { "instruction_tb", VCPU_STAT(instruction_tb) },
135 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
bb25b9ba 136 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
a37cb07a 137 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
95ca2cb5 138 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 139 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 140 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 141 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 142 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 143 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 144 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
145 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 146 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 147 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
148 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 149 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 150 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
151 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
152 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 153 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
154 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
155 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
866c138c
CB		 156 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
157 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
158 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
8474e5ca 159 { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) },
866c138c
CB		 160 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
161 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
162 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
a37cb07a 163 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
b0c632db
HC		 164 { NULL }
165};
166
8fa1696e
CW		 167struct kvm_s390_tod_clock_ext {
168 __u8 epoch_idx;
169 __u64 tod;
170 __u8 reserved[7];
171} __packed;
172
a411edf1
DH		 173/* allow nested virtualization in KVM (if enabled by user space) */
174static int nested;
175module_param(nested, int, S_IRUGO);
176MODULE_PARM_DESC(nested, "Nested virtualization support");
177
a4499382
JF		 178/* allow 1m huge page guest backing, if !nested */
179static int hpage;
180module_param(hpage, int, 0444);
181MODULE_PARM_DESC(hpage, "1m huge page backing support");
b0c632db 182
8b905d28
CB		 183/* maximum percentage of steal time for polling. >100 is treated like 100 */
184static u8 halt_poll_max_steal = 10;
185module_param(halt_poll_max_steal, byte, 0644);
b41fb528 186MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
8b905d28 187
cc674ef2
MM		 188/* if set to true, the GISA will be initialized and used if available */
189static bool use_gisa = true;
190module_param(use_gisa, bool, 0644);
191MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
192
c3b9e3e1
CB		 193/*
194 * For now we handle at most 16 double words as this is what the s390 base
195 * kernel handles and stores in the prefix page. If we ever need to go beyond
196 * this, this requires changes to code, but the external uapi can stay.
197 */
198#define SIZE_INTERNAL 16
199
200/*
201 * Base feature mask that defines default mask for facilities. Consists of the
202 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
203 */
204static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
205/*
206 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
207 * and defines the facilities that can be enabled via a cpu model.
208 */
209static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
210
211static unsigned long kvm_s390_fac_size(void)
78c4b59f 212{
c3b9e3e1
CB		 213 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
214 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
215 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
216 sizeof(S390_lowcore.stfle_fac_list));
217
218 return SIZE_INTERNAL;
78c4b59f
MM		 219}
220
15c9705f
DH		 221/* available cpu features supported by kvm */
222static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 223/* available subfunctions indicated via query / "test bit" */
224static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 225
9d8d5786 226static struct gmap_notifier gmap_notifier;
a3508fbe 227static struct gmap_notifier vsie_gmap_notifier;
78f26131 228debug_info_t *kvm_s390_dbf;
3e6c5568 229debug_info_t *kvm_s390_dbf_uv;
9d8d5786 230
b0c632db 231/* Section: not file related */
13a34e06 232int kvm_arch_hardware_enable(void)
b0c632db
HC		 233{
234 /* every s390 is virtualization enabled ;-) */
10474ae8 235 return 0;
b0c632db
HC		 236}
237
b9904085 238int kvm_arch_check_processor_compat(void *opaque)
f257d6dc
SC		 239{
240 return 0;
241}
242
29b40f10 243/* forward declarations */
414d3b07
MS		 244static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
245 unsigned long end);
29b40f10 246static int sca_switch_to_extended(struct kvm *kvm);
2c70fe44 247
1575767e
DH		 248static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
249{
250 u8 delta_idx = 0;
251
252 /*
253 * The TOD jumps by delta, we have to compensate this by adding
254 * -delta to the epoch.
255 */
256 delta = -delta;
257
258 /* sign-extension - we're adding to signed values below */
259 if ((s64)delta < 0)
260 delta_idx = -1;
261
262 scb->epoch += delta;
263 if (scb->ecd & ECD_MEF) {
264 scb->epdx += delta_idx;
265 if (scb->epoch < delta)
266 scb->epdx += 1;
267 }
268}
269
fdf03650
FZ		 270/*
271 * This callback is executed during stop_machine(). All CPUs are therefore
272 * temporarily stopped. In order not to change guest behavior, we have to
273 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
274 * so a CPU won't be stopped while calculating with the epoch.
275 */
276static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
277 void *v)
278{
279 struct kvm *kvm;
280 struct kvm_vcpu *vcpu;
281 int i;
282 unsigned long long *delta = v;
283
284 list_for_each_entry(kvm, &vm_list, vm_list) {
fdf03650 285 kvm_for_each_vcpu(i, vcpu, kvm) {
1575767e
DH		 286 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
287 if (i == 0) {
288 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
289 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
290 }
db0758b2
DH		 291 if (vcpu->arch.cputm_enabled)
292 vcpu->arch.cputm_start += *delta;
91473b48 293 if (vcpu->arch.vsie_block)
1575767e
DH		 294 kvm_clock_sync_scb(vcpu->arch.vsie_block,
295 *delta);
fdf03650
FZ		 296 }
297 }
298 return NOTIFY_OK;
299}
300
301static struct notifier_block kvm_clock_notifier = {
302 .notifier_call = kvm_clock_sync,
303};
304
b9904085 305int kvm_arch_hardware_setup(void *opaque)
b0c632db 306{
2c70fe44 307 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 308 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 309 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
310 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 311 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
312 &kvm_clock_notifier);
b0c632db
HC		 313 return 0;
314}
315
316void kvm_arch_hardware_unsetup(void)
317{
b2d73b2a 318 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 319 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 320 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
321 &kvm_clock_notifier);
b0c632db
HC		 322}
323
22be5a13
DH		 324static void allow_cpu_feat(unsigned long nr)
325{
326 set_bit_inv(nr, kvm_s390_available_cpu_feat);
327}
328
0a763c78
DH		 329static inline int plo_test_bit(unsigned char nr)
330{
331 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 332 int cc;
0a763c78
DH		 333
334 asm volatile(
335 /* Parameter registers are ignored for "test bit" */
336 " plo 0,0,0,0(0)\n"
337 " ipm %0\n"
338 " srl %0,28\n"
339 : "=d" (cc)
340 : "d" (r0)
341 : "cc");
342 return cc == 0;
343}
344
d0dea733 345static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
d6681397
CB		 346{
347 register unsigned long r0 asm("0") = 0; /* query function */
348 register unsigned long r1 asm("1") = (unsigned long) query;
349
350 asm volatile(
351 /* Parameter regs are ignored */
352 " .insn rrf,%[opc] << 16,2,4,6,0\n"
b1c41ac3 353 :
d6681397 354 : "d" (r0), "a" (r1), [opc] "i" (opcode)
b1c41ac3 355 : "cc", "memory");
d6681397
CB		 356}
357
173aec2d 358#define INSN_SORTL 0xb938
4f45b90e 359#define INSN_DFLTCC 0xb939
173aec2d 360
22be5a13
DH		 361static void kvm_s390_cpu_feat_init(void)
362{
0a763c78
DH		 363 int i;
364
365 for (i = 0; i < 256; ++i) {
366 if (plo_test_bit(i))
367 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
368 }
369
370 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 371 ptff(kvm_s390_available_subfunc.ptff,
372 sizeof(kvm_s390_available_subfunc.ptff),
373 PTFF_QAF);
0a763c78
DH		 374
375 if (test_facility(17)) { /* MSA */
69c0e360
MS		 376 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
377 kvm_s390_available_subfunc.kmac);
378 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
379 kvm_s390_available_subfunc.kmc);
380 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
381 kvm_s390_available_subfunc.km);
382 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
383 kvm_s390_available_subfunc.kimd);
384 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
385 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 386 }
387 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 388 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
389 kvm_s390_available_subfunc.pckmo);
0a763c78 390 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 391 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
392 kvm_s390_available_subfunc.kmctr);
393 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
394 kvm_s390_available_subfunc.kmf);
395 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
396 kvm_s390_available_subfunc.kmo);
397 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
398 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 399 }
400 if (test_facility(57)) /* MSA5 */
985a9d20 401 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 402 kvm_s390_available_subfunc.ppno);
0a763c78 403
e000b8e0
JH		 404 if (test_facility(146)) /* MSA8 */
405 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
406 kvm_s390_available_subfunc.kma);
407
13209ad0
CB		 408 if (test_facility(155)) /* MSA9 */
409 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
410 kvm_s390_available_subfunc.kdsa);
411
173aec2d
CB		 412 if (test_facility(150)) /* SORTL */
413 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
414
4f45b90e
CB		 415 if (test_facility(151)) /* DFLTCC */
416 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
417
22be5a13
DH		 418 if (MACHINE_HAS_ESOP)
419 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 420 /*
421 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
422 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
423 */
424 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 425 !test_facility(3) || !nested)
a3508fbe
DH		 426 return;
427 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 428 if (sclp.has_64bscao)
429 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 430 if (sclp.has_siif)
431 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 432 if (sclp.has_gpere)
433 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 434 if (sclp.has_gsls)
435 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 436 if (sclp.has_ib)
437 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 438 if (sclp.has_cei)
439 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 440 if (sclp.has_ibs)
441 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 442 if (sclp.has_kss)
443 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 444 /*
445 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
446 * all skey handling functions read/set the skey from the PGSTE
447 * instead of the real storage key.
448 *
449 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
450 * pages being detected as preserved although they are resident.
451 *
452 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
453 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
454 *
455 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
456 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
457 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
458 *
459 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
460 * cannot easily shadow the SCA because of the ipte lock.
461 */
22be5a13
DH		 462}
463
b0c632db
HC		 464int kvm_arch_init(void *opaque)
465{
f76f6371 466 int rc = -ENOMEM;
308c3e66 467
78f26131
CB		 468 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
469 if (!kvm_s390_dbf)
470 return -ENOMEM;
471
3e6c5568
JF		 472 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
473 if (!kvm_s390_dbf_uv)
474 goto out;
475
476 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
477 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
f76f6371 478 goto out;
78f26131 479
22be5a13
DH		 480 kvm_s390_cpu_feat_init();
481
84877d93 482 /* Register floating interrupt controller interface. */
308c3e66
MM		 483 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
484 if (rc) {
8d43d570 485 pr_err("A FLIC registration call failed with rc=%d\n", rc);
f76f6371 486 goto out;
308c3e66 487 }
b1d1e76e
MM		 488
489 rc = kvm_s390_gib_init(GAL_ISC);
490 if (rc)
f76f6371 491 goto out;
b1d1e76e 492
308c3e66
MM		 493 return 0;
494
f76f6371
JF		 495out:
496 kvm_arch_exit();
308c3e66 497 return rc;
b0c632db
HC		 498}
499
78f26131
CB		 500void kvm_arch_exit(void)
501{
1282c21e 502 kvm_s390_gib_destroy();
78f26131 503 debug_unregister(kvm_s390_dbf);
3e6c5568 504 debug_unregister(kvm_s390_dbf_uv);
78f26131
CB		 505}
506
b0c632db
HC		 507/* Section: device related */
508long kvm_arch_dev_ioctl(struct file *filp,
509 unsigned int ioctl, unsigned long arg)
510{
511 if (ioctl == KVM_S390_ENABLE_SIE)
512 return s390_enable_sie();
513 return -EINVAL;
514}
515
784aa3d7 516int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 517{
d7b0b5eb
CO		 518 int r;
519
2bd0ac4e 520 switch (ext) {
d7b0b5eb 521 case KVM_CAP_S390_PSW:
b6cf8788 522 case KVM_CAP_S390_GMAP:
52e16b18 523 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 524#ifdef CONFIG_KVM_S390_UCONTROL
525 case KVM_CAP_S390_UCONTROL:
526#endif
3c038e6b 527 case KVM_CAP_ASYNC_PF:
60b413c9 528 case KVM_CAP_SYNC_REGS:
14eebd91 529 case KVM_CAP_ONE_REG:
d6712df9 530 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 531 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 532 case KVM_CAP_IOEVENTFD:
c05c4186 533 case KVM_CAP_DEVICE_CTRL:
78599d90 534 case KVM_CAP_S390_IRQCHIP:
f2061656 535 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 536 case KVM_CAP_MP_STATE:
460df4c1 537 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 538 case KVM_CAP_S390_INJECT_IRQ:
2444b352 539 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 540 case KVM_CAP_S390_USER_STSI:
30ee2a98 541 case KVM_CAP_S390_SKEYS:
816c7667 542 case KVM_CAP_S390_IRQ_STATE:
6502a34c 543 case KVM_CAP_S390_USER_INSTR0:
4036e387 544 case KVM_CAP_S390_CMMA_MIGRATION:
47a4693e 545 case KVM_CAP_S390_AIS:
da9a1446 546 case KVM_CAP_S390_AIS_MIGRATION:
7de3f142 547 case KVM_CAP_S390_VCPU_RESETS:
495907ec 548 case KVM_CAP_SET_GUEST_DEBUG:
d7b0b5eb
CO		 549 r = 1;
550 break;
a4499382
JF		 551 case KVM_CAP_S390_HPAGE_1M:
552 r = 0;
40ebdb8e 553 if (hpage && !kvm_is_ucontrol(kvm))
a4499382
JF		 554 r = 1;
555 break;
41408c28
TH		 556 case KVM_CAP_S390_MEM_OP:
557 r = MEM_OP_MAX_SIZE;
558 break;
e726b1bd
CB		 559 case KVM_CAP_NR_VCPUS:
560 case KVM_CAP_MAX_VCPUS:
a86cb413 561 case KVM_CAP_MAX_VCPU_ID:
76a6dd72 562 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 563 if (!kvm_s390_use_sca_entries())
564 r = KVM_MAX_VCPUS;
565 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 566 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 567 break;
1526bf9c 568 case KVM_CAP_S390_COW:
abf09bed 569 r = MACHINE_HAS_ESOP;
1526bf9c 570 break;
68c55750
EF		 571 case KVM_CAP_S390_VECTOR_REGISTERS:
572 r = MACHINE_HAS_VX;
573 break;
c6e5f166
FZ		 574 case KVM_CAP_S390_RI:
575 r = test_facility(64);
576 break;
4e0b1ab7
FZ		 577 case KVM_CAP_S390_GS:
578 r = test_facility(133);
579 break;
35b3fde6
CB		 580 case KVM_CAP_S390_BPB:
581 r = test_facility(82);
582 break;
13da9ae1
CB		 583 case KVM_CAP_S390_PROTECTED:
584 r = is_prot_virt_host();
585 break;
2bd0ac4e 586 default:
d7b0b5eb 587 r = 0;
2bd0ac4e 588 }
d7b0b5eb 589 return r;
b0c632db
HC		 590}
591
0dff0846 592void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
15f36ebd 593{
0959e168 594 int i;
15f36ebd 595 gfn_t cur_gfn, last_gfn;
0959e168 596 unsigned long gaddr, vmaddr;
15f36ebd 597 struct gmap *gmap = kvm->arch.gmap;
0959e168 598 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
15f36ebd 599
0959e168
JF		 600 /* Loop over all guest segments */
601 cur_gfn = memslot->base_gfn;
15f36ebd 602 last_gfn = memslot->base_gfn + memslot->npages;
0959e168
JF		 603 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
604 gaddr = gfn_to_gpa(cur_gfn);
605 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
606 if (kvm_is_error_hva(vmaddr))
607 continue;
608
609 bitmap_zero(bitmap, _PAGE_ENTRIES);
610 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
611 for (i = 0; i < _PAGE_ENTRIES; i++) {
612 if (test_bit(i, bitmap))
613 mark_page_dirty(kvm, cur_gfn + i);
614 }
15f36ebd 615
1763f8d0
CB		 616 if (fatal_signal_pending(current))
617 return;
70c88a00 618 cond_resched();
15f36ebd 619 }
15f36ebd
JH		 620}
621
b0c632db 622/* Section: vm related */
a6e2f683
ED		 623static void sca_del_vcpu(struct kvm_vcpu *vcpu);
624
b0c632db
HC		 625/*
626 * Get (and clear) the dirty memory log for a memory slot.
627 */
628int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
629 struct kvm_dirty_log *log)
630{
15f36ebd
JH		 631 int r;
632 unsigned long n;
633 struct kvm_memory_slot *memslot;
2a49f61d 634 int is_dirty;
15f36ebd 635
e1e8a962
JF		 636 if (kvm_is_ucontrol(kvm))
637 return -EINVAL;
638
15f36ebd
JH		 639 mutex_lock(&kvm->slots_lock);
640
641 r = -EINVAL;
642 if (log->slot >= KVM_USER_MEM_SLOTS)
643 goto out;
644
2a49f61d 645 r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
15f36ebd
JH		 646 if (r)
647 goto out;
648
649 /* Clear the dirty log */
650 if (is_dirty) {
651 n = kvm_dirty_bitmap_bytes(memslot);
652 memset(memslot->dirty_bitmap, 0, n);
653 }
654 r = 0;
655out:
656 mutex_unlock(&kvm->slots_lock);
657 return r;
b0c632db
HC		 658}
659
6502a34c
DH		 660static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
661{
662 unsigned int i;
663 struct kvm_vcpu *vcpu;
664
665 kvm_for_each_vcpu(i, vcpu, kvm) {
666 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
667 }
668}
669
e5d83c74 670int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
d938dc55
CH		 671{
672 int r;
673
674 if (cap->flags)
675 return -EINVAL;
676
677 switch (cap->cap) {
84223598 678 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 679 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 680 kvm->arch.use_irqchip = 1;
681 r = 0;
682 break;
2444b352 683 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 684 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 685 kvm->arch.user_sigp = 1;
686 r = 0;
687 break;
68c55750 688 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 689 mutex_lock(&kvm->lock);
a03825bb 690 if (kvm->created_vcpus) {
5967c17b
DH		 691 r = -EBUSY;
692 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 693 set_kvm_facility(kvm->arch.model.fac_mask, 129);
694 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 695 if (test_facility(134)) {
696 set_kvm_facility(kvm->arch.model.fac_mask, 134);
697 set_kvm_facility(kvm->arch.model.fac_list, 134);
698 }
53743aa7
MS		 699 if (test_facility(135)) {
700 set_kvm_facility(kvm->arch.model.fac_mask, 135);
701 set_kvm_facility(kvm->arch.model.fac_list, 135);
702 }
7832e91c
CB		 703 if (test_facility(148)) {
704 set_kvm_facility(kvm->arch.model.fac_mask, 148);
705 set_kvm_facility(kvm->arch.model.fac_list, 148);
706 }
d5cb6ab1
CB		 707 if (test_facility(152)) {
708 set_kvm_facility(kvm->arch.model.fac_mask, 152);
709 set_kvm_facility(kvm->arch.model.fac_list, 152);
710 }
18280d8b
MM		 711 r = 0;
712 } else
713 r = -EINVAL;
5967c17b 714 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 715 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
716 r ? "(not available)" : "(success)");
68c55750 717 break;
c6e5f166
FZ		 718 case KVM_CAP_S390_RI:
719 r = -EINVAL;
720 mutex_lock(&kvm->lock);
a03825bb 721 if (kvm->created_vcpus) {
c6e5f166
FZ		 722 r = -EBUSY;
723 } else if (test_facility(64)) {
c54f0d6a
DH		 724 set_kvm_facility(kvm->arch.model.fac_mask, 64);
725 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 726 r = 0;
727 }
728 mutex_unlock(&kvm->lock);
729 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
730 r ? "(not available)" : "(success)");
731 break;
47a4693e
YMZ		 732 case KVM_CAP_S390_AIS:
733 mutex_lock(&kvm->lock);
734 if (kvm->created_vcpus) {
735 r = -EBUSY;
736 } else {
737 set_kvm_facility(kvm->arch.model.fac_mask, 72);
738 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 739 r = 0;
740 }
741 mutex_unlock(&kvm->lock);
742 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
743 r ? "(not available)" : "(success)");
744 break;
4e0b1ab7
FZ		 745 case KVM_CAP_S390_GS:
746 r = -EINVAL;
747 mutex_lock(&kvm->lock);
241e3ec0 748 if (kvm->created_vcpus) {
4e0b1ab7
FZ		 749 r = -EBUSY;
750 } else if (test_facility(133)) {
751 set_kvm_facility(kvm->arch.model.fac_mask, 133);
752 set_kvm_facility(kvm->arch.model.fac_list, 133);
753 r = 0;
754 }
755 mutex_unlock(&kvm->lock);
756 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
757 r ? "(not available)" : "(success)");
758 break;
a4499382
JF		 759 case KVM_CAP_S390_HPAGE_1M:
760 mutex_lock(&kvm->lock);
761 if (kvm->created_vcpus)
762 r = -EBUSY;
40ebdb8e 763 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
a4499382
JF		 764 r = -EINVAL;
765 else {
766 r = 0;
df88f318 767 down_write(&kvm->mm->mmap_sem);
a4499382 768 kvm->mm->context.allow_gmap_hpage_1m = 1;
df88f318 769 up_write(&kvm->mm->mmap_sem);
a4499382
JF		 770 /*
771 * We might have to create fake 4k page
772 * tables. To avoid that the hardware works on
773 * stale PGSTEs, we emulate these instructions.
774 */
775 kvm->arch.use_skf = 0;
776 kvm->arch.use_pfmfi = 0;
777 }
778 mutex_unlock(&kvm->lock);
779 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
780 r ? "(not available)" : "(success)");
781 break;
e44fc8c9 782 case KVM_CAP_S390_USER_STSI:
c92ea7b9 783 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 784 kvm->arch.user_stsi = 1;
785 r = 0;
786 break;
6502a34c
DH		 787 case KVM_CAP_S390_USER_INSTR0:
788 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
789 kvm->arch.user_instr0 = 1;
790 icpt_operexc_on_all_vcpus(kvm);
791 r = 0;
792 break;
d938dc55
CH		 793 default:
794 r = -EINVAL;
795 break;
796 }
797 return r;
798}
799
8c0a7ce6
DD		 800static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
801{
802 int ret;
803
804 switch (attr->attr) {
805 case KVM_S390_VM_MEM_LIMIT_SIZE:
806 ret = 0;
c92ea7b9 807 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 808 kvm->arch.mem_limit);
809 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 810 ret = -EFAULT;
811 break;
812 default:
813 ret = -ENXIO;
814 break;
815 }
816 return ret;
817}
818
819static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 820{
821 int ret;
822 unsigned int idx;
823 switch (attr->attr) {
824 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 825 ret = -ENXIO;
c24cc9c8 826 if (!sclp.has_cmma)
e6db1d61
DD		 827 break;
828
c92ea7b9 829 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 830 mutex_lock(&kvm->lock);
a4499382
JF		 831 if (kvm->created_vcpus)
832 ret = -EBUSY;
833 else if (kvm->mm->context.allow_gmap_hpage_1m)
834 ret = -EINVAL;
835 else {
4f718eab 836 kvm->arch.use_cmma = 1;
c9f0a2b8
JF		 837 /* Not compatible with cmma. */
838 kvm->arch.use_pfmfi = 0;
4f718eab
DD		 839 ret = 0;
840 }
841 mutex_unlock(&kvm->lock);
842 break;
843 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 844 ret = -ENXIO;
845 if (!sclp.has_cmma)
846 break;
c3489155
DD		 847 ret = -EINVAL;
848 if (!kvm->arch.use_cmma)
849 break;
850
c92ea7b9 851 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 852 mutex_lock(&kvm->lock);
853 idx = srcu_read_lock(&kvm->srcu);
a13cff31 854 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 855 srcu_read_unlock(&kvm->srcu, idx);
856 mutex_unlock(&kvm->lock);
857 ret = 0;
858 break;
8c0a7ce6
DD		 859 case KVM_S390_VM_MEM_LIMIT_SIZE: {
860 unsigned long new_limit;
861
862 if (kvm_is_ucontrol(kvm))
863 return -EINVAL;
864
865 if (get_user(new_limit, (u64 __user *)attr->addr))
866 return -EFAULT;
867
a3a92c31
DD		 868 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
869 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 870 return -E2BIG;
871
a3a92c31
DD		 872 if (!new_limit)
873 return -EINVAL;
874
6ea427bb 875 /* gmap_create takes last usable address */
a3a92c31
DD		 876 if (new_limit != KVM_S390_NO_MEM_LIMIT)
877 new_limit -= 1;
878
8c0a7ce6
DD		 879 ret = -EBUSY;
880 mutex_lock(&kvm->lock);
a03825bb 881 if (!kvm->created_vcpus) {
6ea427bb
MS		 882 /* gmap_create will round the limit up */
883 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 884
885 if (!new) {
886 ret = -ENOMEM;
887 } else {
6ea427bb 888 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 889 new->private = kvm;
890 kvm->arch.gmap = new;
891 ret = 0;
892 }
893 }
894 mutex_unlock(&kvm->lock);
a3a92c31
DD		 895 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
896 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
897 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 898 break;
899 }
4f718eab
DD		 900 default:
901 ret = -ENXIO;
902 break;
903 }
904 return ret;
905}
906
a374e892
TK		 907static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
908
20c922f0 909void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
a374e892
TK		 910{
911 struct kvm_vcpu *vcpu;
912 int i;
913
20c922f0
TK		 914 kvm_s390_vcpu_block_all(kvm);
915
3194cdb7 916 kvm_for_each_vcpu(i, vcpu, kvm) {
20c922f0 917 kvm_s390_vcpu_crypto_setup(vcpu);
3194cdb7
DH		 918 /* recreate the shadow crycb by leaving the VSIE handler */
919 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
920 }
20c922f0
TK		 921
922 kvm_s390_vcpu_unblock_all(kvm);
923}
924
925static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
926{
a374e892
TK		 927 mutex_lock(&kvm->lock);
928 switch (attr->attr) {
929 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
8e41bd54
CB		 930 if (!test_kvm_facility(kvm, 76)) {
931 mutex_unlock(&kvm->lock);
37940fb0 932 return -EINVAL;
8e41bd54 933 }
a374e892
TK		 934 get_random_bytes(
935 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
936 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
937 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 938 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 939 break;
940 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
8e41bd54
CB		 941 if (!test_kvm_facility(kvm, 76)) {
942 mutex_unlock(&kvm->lock);
37940fb0 943 return -EINVAL;
8e41bd54 944 }
a374e892
TK		 945 get_random_bytes(
946 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
947 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
948 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 949 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 950 break;
951 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
8e41bd54
CB		 952 if (!test_kvm_facility(kvm, 76)) {
953 mutex_unlock(&kvm->lock);
37940fb0 954 return -EINVAL;
8e41bd54 955 }
a374e892
TK		 956 kvm->arch.crypto.aes_kw = 0;
957 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
958 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 959 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 960 break;
961 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
8e41bd54
CB		 962 if (!test_kvm_facility(kvm, 76)) {
963 mutex_unlock(&kvm->lock);
37940fb0 964 return -EINVAL;
8e41bd54 965 }
a374e892
TK		 966 kvm->arch.crypto.dea_kw = 0;
967 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
968 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 969 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892 970 break;
37940fb0
TK		 971 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
972 if (!ap_instructions_available()) {
973 mutex_unlock(&kvm->lock);
974 return -EOPNOTSUPP;
975 }
976 kvm->arch.crypto.apie = 1;
977 break;
978 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
979 if (!ap_instructions_available()) {
980 mutex_unlock(&kvm->lock);
981 return -EOPNOTSUPP;
982 }
983 kvm->arch.crypto.apie = 0;
984 break;
a374e892
TK		 985 default:
986 mutex_unlock(&kvm->lock);
987 return -ENXIO;
988 }
989
20c922f0 990 kvm_s390_vcpu_crypto_reset_all(kvm);
a374e892
TK		 991 mutex_unlock(&kvm->lock);
992 return 0;
993}
994
190df4a2
CI		 995static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
996{
997 int cx;
998 struct kvm_vcpu *vcpu;
999
1000 kvm_for_each_vcpu(cx, vcpu, kvm)
1001 kvm_s390_sync_request(req, vcpu);
1002}
1003
1004/*
1005 * Must be called with kvm->srcu held to avoid races on memslots, and with
1de1ea7e 1006 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
190df4a2
CI		 1007 */
1008static int kvm_s390_vm_start_migration(struct kvm *kvm)
1009{
190df4a2 1010 struct kvm_memory_slot *ms;
190df4a2 1011 struct kvm_memslots *slots;
afdad616 1012 unsigned long ram_pages = 0;
190df4a2
CI		 1013 int slotnr;
1014
1015 /* migration mode already enabled */
afdad616 1016 if (kvm->arch.migration_mode)
190df4a2 1017 return 0;
190df4a2
CI		 1018 slots = kvm_memslots(kvm);
1019 if (!slots || !slots->used_slots)
1020 return -EINVAL;
1021
afdad616
CI		 1022 if (!kvm->arch.use_cmma) {
1023 kvm->arch.migration_mode = 1;
1024 return 0;
1025 }
1026 /* mark all the pages in active slots as dirty */
1027 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1028 ms = slots->memslots + slotnr;
13a17cc0
IM		 1029 if (!ms->dirty_bitmap)
1030 return -EINVAL;
190df4a2 1031 /*
afdad616
CI		 1032 * The second half of the bitmap is only used on x86,
1033 * and would be wasted otherwise, so we put it to good
1034 * use here to keep track of the state of the storage
1035 * attributes.
190df4a2 1036 */
afdad616
CI		 1037 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1038 ram_pages += ms->npages;
190df4a2 1039 }
afdad616
CI		 1040 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1041 kvm->arch.migration_mode = 1;
1042 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
190df4a2
CI		 1043 return 0;
1044}
1045
1046/*
1de1ea7e 1047 * Must be called with kvm->slots_lock to avoid races with ourselves and
190df4a2
CI		 1048 * kvm_s390_vm_start_migration.
1049 */
1050static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1051{
190df4a2 1052 /* migration mode already disabled */
afdad616 1053 if (!kvm->arch.migration_mode)
190df4a2 1054 return 0;
afdad616
CI		 1055 kvm->arch.migration_mode = 0;
1056 if (kvm->arch.use_cmma)
190df4a2 1057 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
190df4a2
CI		 1058 return 0;
1059}
1060
1061static int kvm_s390_vm_set_migration(struct kvm *kvm,
1062 struct kvm_device_attr *attr)
1063{
1de1ea7e 1064 int res = -ENXIO;
190df4a2 1065
1de1ea7e 1066 mutex_lock(&kvm->slots_lock);
190df4a2
CI		 1067 switch (attr->attr) {
1068 case KVM_S390_VM_MIGRATION_START:
190df4a2 1069 res = kvm_s390_vm_start_migration(kvm);
190df4a2
CI		 1070 break;
1071 case KVM_S390_VM_MIGRATION_STOP:
1072 res = kvm_s390_vm_stop_migration(kvm);
1073 break;
1074 default:
1075 break;
1076 }
1de1ea7e 1077 mutex_unlock(&kvm->slots_lock);
190df4a2
CI		 1078
1079 return res;
1080}
1081
1082static int kvm_s390_vm_get_migration(struct kvm *kvm,
1083 struct kvm_device_attr *attr)
1084{
afdad616 1085 u64 mig = kvm->arch.migration_mode;
190df4a2
CI		 1086
1087 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1088 return -ENXIO;
1089
1090 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1091 return -EFAULT;
1092 return 0;
1093}
1094
8fa1696e
CW		 1095static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1096{
1097 struct kvm_s390_vm_tod_clock gtod;
1098
1099 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1100 return -EFAULT;
1101
0e7def5f 1102 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
8fa1696e 1103 return -EINVAL;
0e7def5f 1104 kvm_s390_set_tod_clock(kvm, &gtod);
8fa1696e
CW		 1105
1106 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1107 gtod.epoch_idx, gtod.tod);
1108
1109 return 0;
1110}
1111
72f25020
JH		 1112static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1113{
1114 u8 gtod_high;
1115
1116 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1117 sizeof(gtod_high)))
1118 return -EFAULT;
1119
1120 if (gtod_high != 0)
1121 return -EINVAL;
58c383c6 1122 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1123
1124 return 0;
1125}
1126
1127static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1128{
0e7def5f 1129 struct kvm_s390_vm_tod_clock gtod = { 0 };
72f25020 1130
0e7def5f
DH		 1131 if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1132 sizeof(gtod.tod)))
72f25020
JH		 1133 return -EFAULT;
1134
0e7def5f
DH		 1135 kvm_s390_set_tod_clock(kvm, &gtod);
1136 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
72f25020
JH		 1137 return 0;
1138}
1139
1140static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1141{
1142 int ret;
1143
1144 if (attr->flags)
1145 return -EINVAL;
1146
1147 switch (attr->attr) {
8fa1696e
CW		 1148 case KVM_S390_VM_TOD_EXT:
1149 ret = kvm_s390_set_tod_ext(kvm, attr);
1150 break;
72f25020
JH		 1151 case KVM_S390_VM_TOD_HIGH:
1152 ret = kvm_s390_set_tod_high(kvm, attr);
1153 break;
1154 case KVM_S390_VM_TOD_LOW:
1155 ret = kvm_s390_set_tod_low(kvm, attr);
1156 break;
1157 default:
1158 ret = -ENXIO;
1159 break;
1160 }
1161 return ret;
1162}
1163
33d1b272
DH		 1164static void kvm_s390_get_tod_clock(struct kvm *kvm,
1165 struct kvm_s390_vm_tod_clock *gtod)
8fa1696e
CW		 1166{
1167 struct kvm_s390_tod_clock_ext htod;
1168
1169 preempt_disable();
1170
1171 get_tod_clock_ext((char *)&htod);
1172
1173 gtod->tod = htod.tod + kvm->arch.epoch;
33d1b272
DH		 1174 gtod->epoch_idx = 0;
1175 if (test_kvm_facility(kvm, 139)) {
1176 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1177 if (gtod->tod < htod.tod)
1178 gtod->epoch_idx += 1;
1179 }
8fa1696e
CW		 1180
1181 preempt_enable();
1182}
1183
1184static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1185{
1186 struct kvm_s390_vm_tod_clock gtod;
1187
1188 memset(&gtod, 0, sizeof(gtod));
33d1b272 1189 kvm_s390_get_tod_clock(kvm, &gtod);
8fa1696e
CW		 1190 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1191 return -EFAULT;
1192
1193 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1194 gtod.epoch_idx, gtod.tod);
1195 return 0;
1196}
1197
72f25020
JH		 1198static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1199{
1200 u8 gtod_high = 0;
1201
1202 if (copy_to_user((void __user *)attr->addr, &gtod_high,
1203 sizeof(gtod_high)))
1204 return -EFAULT;
58c383c6 1205 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1206
1207 return 0;
1208}
1209
1210static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1211{
5a3d883a 1212 u64 gtod;
72f25020 1213
60417fcc 1214 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 1215 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1216 return -EFAULT;
58c383c6 1217 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 1218
1219 return 0;
1220}
1221
1222static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1223{
1224 int ret;
1225
1226 if (attr->flags)
1227 return -EINVAL;
1228
1229 switch (attr->attr) {
8fa1696e
CW		 1230 case KVM_S390_VM_TOD_EXT:
1231 ret = kvm_s390_get_tod_ext(kvm, attr);
1232 break;
72f25020
JH		 1233 case KVM_S390_VM_TOD_HIGH:
1234 ret = kvm_s390_get_tod_high(kvm, attr);
1235 break;
1236 case KVM_S390_VM_TOD_LOW:
1237 ret = kvm_s390_get_tod_low(kvm, attr);
1238 break;
1239 default:
1240 ret = -ENXIO;
1241 break;
1242 }
1243 return ret;
1244}
1245
658b6eda
MM		 1246static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1247{
1248 struct kvm_s390_vm_cpu_processor *proc;
053dd230 1249 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 1250 int ret = 0;
1251
1252 mutex_lock(&kvm->lock);
a03825bb 1253 if (kvm->created_vcpus) {
658b6eda
MM		 1254 ret = -EBUSY;
1255 goto out;
1256 }
1257 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1258 if (!proc) {
1259 ret = -ENOMEM;
1260 goto out;
1261 }
1262 if (!copy_from_user(proc, (void __user *)attr->addr,
1263 sizeof(*proc))) {
9bb0ec09 1264 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 1265 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1266 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 1267 if (lowest_ibc && proc->ibc) {
053dd230
DH		 1268 if (proc->ibc > unblocked_ibc)
1269 kvm->arch.model.ibc = unblocked_ibc;
1270 else if (proc->ibc < lowest_ibc)
1271 kvm->arch.model.ibc = lowest_ibc;
1272 else
1273 kvm->arch.model.ibc = proc->ibc;
1274 }
c54f0d6a 1275 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 1276 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1277 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1278 kvm->arch.model.ibc,
1279 kvm->arch.model.cpuid);
1280 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1281 kvm->arch.model.fac_list[0],
1282 kvm->arch.model.fac_list[1],
1283 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1284 } else
1285 ret = -EFAULT;
1286 kfree(proc);
1287out:
1288 mutex_unlock(&kvm->lock);
1289 return ret;
1290}
1291
15c9705f
DH		 1292static int kvm_s390_set_processor_feat(struct kvm *kvm,
1293 struct kvm_device_attr *attr)
1294{
1295 struct kvm_s390_vm_cpu_feat data;
15c9705f
DH		 1296
1297 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1298 return -EFAULT;
1299 if (!bitmap_subset((unsigned long *) data.feat,
1300 kvm_s390_available_cpu_feat,
1301 KVM_S390_VM_CPU_FEAT_NR_BITS))
1302 return -EINVAL;
1303
1304 mutex_lock(&kvm->lock);
2f8311c9
CB		 1305 if (kvm->created_vcpus) {
1306 mutex_unlock(&kvm->lock);
1307 return -EBUSY;
15c9705f 1308 }
2f8311c9
CB		 1309 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1310 KVM_S390_VM_CPU_FEAT_NR_BITS);
15c9705f 1311 mutex_unlock(&kvm->lock);
2f8311c9
CB		 1312 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1313 data.feat[0],
1314 data.feat[1],
1315 data.feat[2]);
1316 return 0;
15c9705f
DH		 1317}
1318
0a763c78
DH		 1319static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1320 struct kvm_device_attr *attr)
1321{
346fa2f8
CB		 1322 mutex_lock(&kvm->lock);
1323 if (kvm->created_vcpus) {
1324 mutex_unlock(&kvm->lock);
1325 return -EBUSY;
1326 }
1327
1328 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1329 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1330 mutex_unlock(&kvm->lock);
1331 return -EFAULT;
1332 }
1333 mutex_unlock(&kvm->lock);
1334
11ba5961
CB		 1335 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1336 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1337 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1338 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1339 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1340 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1341 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1342 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1343 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1344 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1345 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1346 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1347 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1348 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1349 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1350 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1351 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1352 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1353 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1354 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1355 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1356 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1357 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1358 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1359 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1360 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1361 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1362 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1363 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1364 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1365 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1366 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1367 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1368 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1369 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1370 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1371 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1372 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1373 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1374 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1375 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1376 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1377 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1378 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
13209ad0
CB		 1379 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1380 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1381 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
173aec2d
CB		 1382 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1383 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1384 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1385 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1386 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
4f45b90e
CB		 1387 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1388 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1389 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1390 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1391 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
11ba5961 1392
346fa2f8 1393 return 0;
0a763c78
DH		 1394}
1395
658b6eda
MM		 1396static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1397{
1398 int ret = -ENXIO;
1399
1400 switch (attr->attr) {
1401 case KVM_S390_VM_CPU_PROCESSOR:
1402 ret = kvm_s390_set_processor(kvm, attr);
1403 break;
15c9705f
DH		 1404 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1405 ret = kvm_s390_set_processor_feat(kvm, attr);
1406 break;
0a763c78
DH		 1407 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1408 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1409 break;
658b6eda
MM		 1410 }
1411 return ret;
1412}
1413
1414static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1415{
1416 struct kvm_s390_vm_cpu_processor *proc;
1417 int ret = 0;
1418
1419 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1420 if (!proc) {
1421 ret = -ENOMEM;
1422 goto out;
1423 }
9bb0ec09 1424 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 1425 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 1426 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1427 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1428 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1429 kvm->arch.model.ibc,
1430 kvm->arch.model.cpuid);
1431 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1432 kvm->arch.model.fac_list[0],
1433 kvm->arch.model.fac_list[1],
1434 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1435 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1436 ret = -EFAULT;
1437 kfree(proc);
1438out:
1439 return ret;
1440}
1441
1442static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1443{
1444 struct kvm_s390_vm_cpu_machine *mach;
1445 int ret = 0;
1446
1447 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1448 if (!mach) {
1449 ret = -ENOMEM;
1450 goto out;
1451 }
1452 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 1453 mach->ibc = sclp.ibc;
c54f0d6a 1454 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 1455 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 1456 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 1457 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 1458 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1459 kvm->arch.model.ibc,
1460 kvm->arch.model.cpuid);
1461 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1462 mach->fac_mask[0],
1463 mach->fac_mask[1],
1464 mach->fac_mask[2]);
1465 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1466 mach->fac_list[0],
1467 mach->fac_list[1],
1468 mach->fac_list[2]);
658b6eda
MM		 1469 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1470 ret = -EFAULT;
1471 kfree(mach);
1472out:
1473 return ret;
1474}
1475
15c9705f
DH		 1476static int kvm_s390_get_processor_feat(struct kvm *kvm,
1477 struct kvm_device_attr *attr)
1478{
1479 struct kvm_s390_vm_cpu_feat data;
1480
1481 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1482 KVM_S390_VM_CPU_FEAT_NR_BITS);
1483 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1484 return -EFAULT;
2f8311c9
CB		 1485 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1486 data.feat[0],
1487 data.feat[1],
1488 data.feat[2]);
15c9705f
DH		 1489 return 0;
1490}
1491
1492static int kvm_s390_get_machine_feat(struct kvm *kvm,
1493 struct kvm_device_attr *attr)
1494{
1495 struct kvm_s390_vm_cpu_feat data;
1496
1497 bitmap_copy((unsigned long *) data.feat,
1498 kvm_s390_available_cpu_feat,
1499 KVM_S390_VM_CPU_FEAT_NR_BITS);
1500 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1501 return -EFAULT;
2f8311c9
CB		 1502 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1503 data.feat[0],
1504 data.feat[1],
1505 data.feat[2]);
15c9705f
DH		 1506 return 0;
1507}
1508
0a763c78
DH		 1509static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1510 struct kvm_device_attr *attr)
1511{
346fa2f8
CB		 1512 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1513 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1514 return -EFAULT;
1515
11ba5961
CB		 1516 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1517 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1518 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1519 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1520 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1521 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1522 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1523 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1524 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1525 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1526 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1527 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1528 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1529 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1530 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1531 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1532 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1533 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1534 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1535 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1536 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1537 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1538 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1539 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1540 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1541 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1542 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1543 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1544 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1545 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1546 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1547 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1548 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1549 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1550 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1551 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1552 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1553 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1554 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1555 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1556 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1557 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1558 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1559 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
13209ad0
CB		 1560 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1561 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1562 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
173aec2d
CB		 1563 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1564 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1565 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1566 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1567 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
4f45b90e
CB		 1568 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1569 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1570 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1571 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1572 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
11ba5961 1573
346fa2f8 1574 return 0;
0a763c78
DH		 1575}
1576
1577static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1578 struct kvm_device_attr *attr)
1579{
1580 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1581 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1582 return -EFAULT;
11ba5961
CB		 1583
1584 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1585 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1586 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1587 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1588 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1589 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1590 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1591 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1592 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1593 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1594 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1595 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1596 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1597 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1598 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1599 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1600 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1601 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1602 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1603 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1604 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1605 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1606 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1607 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1608 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1609 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1610 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1611 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1612 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1613 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1614 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1615 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1616 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1617 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1618 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1619 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1620 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1621 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1622 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1623 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1624 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1625 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1626 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1627 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
13209ad0
CB		 1628 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1629 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1630 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
173aec2d
CB		 1631 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1632 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1633 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1634 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1635 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
4f45b90e
CB		 1636 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1637 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1638 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1639 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1640 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
11ba5961 1641
0a763c78
DH		 1642 return 0;
1643}
346fa2f8 1644
658b6eda
MM		 1645static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1646{
1647 int ret = -ENXIO;
1648
1649 switch (attr->attr) {
1650 case KVM_S390_VM_CPU_PROCESSOR:
1651 ret = kvm_s390_get_processor(kvm, attr);
1652 break;
1653 case KVM_S390_VM_CPU_MACHINE:
1654 ret = kvm_s390_get_machine(kvm, attr);
1655 break;
15c9705f
DH		 1656 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1657 ret = kvm_s390_get_processor_feat(kvm, attr);
1658 break;
1659 case KVM_S390_VM_CPU_MACHINE_FEAT:
1660 ret = kvm_s390_get_machine_feat(kvm, attr);
1661 break;
0a763c78
DH		 1662 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1663 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1664 break;
1665 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1666 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1667 break;
658b6eda
MM		 1668 }
1669 return ret;
1670}
1671
f2061656
DD		 1672static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1673{
1674 int ret;
1675
1676 switch (attr->group) {
4f718eab 1677 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1678 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1679 break;
72f25020
JH		 1680 case KVM_S390_VM_TOD:
1681 ret = kvm_s390_set_tod(kvm, attr);
1682 break;
658b6eda
MM		 1683 case KVM_S390_VM_CPU_MODEL:
1684 ret = kvm_s390_set_cpu_model(kvm, attr);
1685 break;
a374e892
TK		 1686 case KVM_S390_VM_CRYPTO:
1687 ret = kvm_s390_vm_set_crypto(kvm, attr);
1688 break;
190df4a2
CI		 1689 case KVM_S390_VM_MIGRATION:
1690 ret = kvm_s390_vm_set_migration(kvm, attr);
1691 break;
f2061656
DD		 1692 default:
1693 ret = -ENXIO;
1694 break;
1695 }
1696
1697 return ret;
1698}
1699
1700static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1701{
8c0a7ce6
DD		 1702 int ret;
1703
1704 switch (attr->group) {
1705 case KVM_S390_VM_MEM_CTRL:
1706 ret = kvm_s390_get_mem_control(kvm, attr);
1707 break;
72f25020
JH		 1708 case KVM_S390_VM_TOD:
1709 ret = kvm_s390_get_tod(kvm, attr);
1710 break;
658b6eda
MM		 1711 case KVM_S390_VM_CPU_MODEL:
1712 ret = kvm_s390_get_cpu_model(kvm, attr);
1713 break;
190df4a2
CI		 1714 case KVM_S390_VM_MIGRATION:
1715 ret = kvm_s390_vm_get_migration(kvm, attr);
1716 break;
8c0a7ce6
DD		 1717 default:
1718 ret = -ENXIO;
1719 break;
1720 }
1721
1722 return ret;
f2061656
DD		 1723}
1724
1725static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1726{
1727 int ret;
1728
1729 switch (attr->group) {
4f718eab
DD		 1730 case KVM_S390_VM_MEM_CTRL:
1731 switch (attr->attr) {
1732 case KVM_S390_VM_MEM_ENABLE_CMMA:
1733 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1734 ret = sclp.has_cmma ? 0 : -ENXIO;
1735 break;
8c0a7ce6 1736 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1737 ret = 0;
1738 break;
1739 default:
1740 ret = -ENXIO;
1741 break;
1742 }
1743 break;
72f25020
JH		 1744 case KVM_S390_VM_TOD:
1745 switch (attr->attr) {
1746 case KVM_S390_VM_TOD_LOW:
1747 case KVM_S390_VM_TOD_HIGH:
1748 ret = 0;
1749 break;
1750 default:
1751 ret = -ENXIO;
1752 break;
1753 }
1754 break;
658b6eda
MM		 1755 case KVM_S390_VM_CPU_MODEL:
1756 switch (attr->attr) {
1757 case KVM_S390_VM_CPU_PROCESSOR:
1758 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1759 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1760 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1761 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
346fa2f8 1762 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1763 ret = 0;
1764 break;
1765 default:
1766 ret = -ENXIO;
1767 break;
1768 }
1769 break;
a374e892
TK		 1770 case KVM_S390_VM_CRYPTO:
1771 switch (attr->attr) {
1772 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1773 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1774 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1775 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1776 ret = 0;
1777 break;
37940fb0
TK		 1778 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1779 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1780 ret = ap_instructions_available() ? 0 : -ENXIO;
1781 break;
a374e892
TK		 1782 default:
1783 ret = -ENXIO;
1784 break;
1785 }
1786 break;
190df4a2
CI		 1787 case KVM_S390_VM_MIGRATION:
1788 ret = 0;
1789 break;
f2061656
DD		 1790 default:
1791 ret = -ENXIO;
1792 break;
1793 }
1794
1795 return ret;
1796}
1797
30ee2a98
JH		 1798static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1799{
1800 uint8_t *keys;
1801 uint64_t hva;
4f899147 1802 int srcu_idx, i, r = 0;
30ee2a98
JH		 1803
1804 if (args->flags != 0)
1805 return -EINVAL;
1806
1807 /* Is this guest using storage keys? */
55531b74 1808 if (!mm_uses_skeys(current->mm))
30ee2a98
JH		 1809 return KVM_S390_GET_SKEYS_NONE;
1810
1811 /* Enforce sane limit on memory allocation */
1812 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1813 return -EINVAL;
1814
752ade68 1815 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1816 if (!keys)
1817 return -ENOMEM;
1818
d3ed1cee 1819 down_read(&current->mm->mmap_sem);
4f899147 1820 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1821 for (i = 0; i < args->count; i++) {
1822 hva = gfn_to_hva(kvm, args->start_gfn + i);
1823 if (kvm_is_error_hva(hva)) {
1824 r = -EFAULT;
d3ed1cee 1825 break;
30ee2a98
JH		 1826 }
1827
154c8c19
DH		 1828 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1829 if (r)
d3ed1cee 1830 break;
30ee2a98 1831 }
4f899147 1832 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee
MS		 1833 up_read(&current->mm->mmap_sem);
1834
1835 if (!r) {
1836 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1837 sizeof(uint8_t) * args->count);
1838 if (r)
1839 r = -EFAULT;
30ee2a98
JH		 1840 }
1841
30ee2a98
JH		 1842 kvfree(keys);
1843 return r;
1844}
1845
1846static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1847{
1848 uint8_t *keys;
1849 uint64_t hva;
4f899147 1850 int srcu_idx, i, r = 0;
bd096f64 1851 bool unlocked;
30ee2a98
JH		 1852
1853 if (args->flags != 0)
1854 return -EINVAL;
1855
1856 /* Enforce sane limit on memory allocation */
1857 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1858 return -EINVAL;
1859
752ade68 1860 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1861 if (!keys)
1862 return -ENOMEM;
1863
1864 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1865 sizeof(uint8_t) * args->count);
1866 if (r) {
1867 r = -EFAULT;
1868 goto out;
1869 }
1870
1871 /* Enable storage key handling for the guest */
14d4a425
DD		 1872 r = s390_enable_skey();
1873 if (r)
1874 goto out;
30ee2a98 1875
bd096f64 1876 i = 0;
d3ed1cee 1877 down_read(&current->mm->mmap_sem);
4f899147 1878 srcu_idx = srcu_read_lock(&kvm->srcu);
bd096f64
JF		 1879 while (i < args->count) {
1880 unlocked = false;
30ee2a98
JH		 1881 hva = gfn_to_hva(kvm, args->start_gfn + i);
1882 if (kvm_is_error_hva(hva)) {
1883 r = -EFAULT;
d3ed1cee 1884 break;
30ee2a98
JH		 1885 }
1886
1887 /* Lowest order bit is reserved */
1888 if (keys[i] & 0x01) {
1889 r = -EINVAL;
d3ed1cee 1890 break;
30ee2a98
JH		 1891 }
1892
fe69eabf 1893 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
bd096f64
JF		 1894 if (r) {
1895 r = fixup_user_fault(current, current->mm, hva,
1896 FAULT_FLAG_WRITE, &unlocked);
1897 if (r)
1898 break;
1899 }
1900 if (!r)
1901 i++;
30ee2a98 1902 }
4f899147 1903 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee 1904 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1905out:
1906 kvfree(keys);
1907 return r;
1908}
1909
4036e387
CI		 1910/*
1911 * Base address and length must be sent at the start of each block, therefore
1912 * it's cheaper to send some clean data, as long as it's less than the size of
1913 * two longs.
1914 */
1915#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1916/* for consistency */
1917#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1918
afdad616
CI		 1919/*
1920 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1921 * address falls in a hole. In that case the index of one of the memslots
1922 * bordering the hole is returned.
1923 */
1924static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1925{
1926 int start = 0, end = slots->used_slots;
1927 int slot = atomic_read(&slots->lru_slot);
1928 struct kvm_memory_slot *memslots = slots->memslots;
1929
1930 if (gfn >= memslots[slot].base_gfn &&
1931 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1932 return slot;
1933
1934 while (start < end) {
1935 slot = start + (end - start) / 2;
1936
1937 if (gfn >= memslots[slot].base_gfn)
1938 end = slot;
1939 else
1940 start = slot + 1;
1941 }
1942
97daa028
SC		 1943 if (start >= slots->used_slots)
1944 return slots->used_slots - 1;
1945
afdad616
CI		 1946 if (gfn >= memslots[start].base_gfn &&
1947 gfn < memslots[start].base_gfn + memslots[start].npages) {
1948 atomic_set(&slots->lru_slot, start);
1949 }
1950
1951 return start;
1952}
1953
1954static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1955 u8 *res, unsigned long bufsize)
1956{
1957 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1958
1959 args->count = 0;
1960 while (args->count < bufsize) {
1961 hva = gfn_to_hva(kvm, cur_gfn);
1962 /*
1963 * We return an error if the first value was invalid, but we
1964 * return successfully if at least one value was copied.
1965 */
1966 if (kvm_is_error_hva(hva))
1967 return args->count ? 0 : -EFAULT;
1968 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1969 pgstev = 0;
1970 res[args->count++] = (pgstev >> 24) & 0x43;
1971 cur_gfn++;
1972 }
1973
1974 return 0;
1975}
1976
1977static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1978 unsigned long cur_gfn)
1979{
1980 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1981 struct kvm_memory_slot *ms = slots->memslots + slotidx;
1982 unsigned long ofs = cur_gfn - ms->base_gfn;
1983
1984 if (ms->base_gfn + ms->npages <= cur_gfn) {
1985 slotidx--;
1986 /* If we are above the highest slot, wrap around */
1987 if (slotidx < 0)
1988 slotidx = slots->used_slots - 1;
1989
1990 ms = slots->memslots + slotidx;
1991 ofs = 0;
1992 }
1993 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1994 while ((slotidx > 0) && (ofs >= ms->npages)) {
1995 slotidx--;
1996 ms = slots->memslots + slotidx;
1997 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1998 }
1999 return ms->base_gfn + ofs;
2000}
2001
2002static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2003 u8 *res, unsigned long bufsize)
2004{
2005 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2006 struct kvm_memslots *slots = kvm_memslots(kvm);
2007 struct kvm_memory_slot *ms;
2008
0774a964
SC		 2009 if (unlikely(!slots->used_slots))
2010 return 0;
2011
afdad616
CI		 2012 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2013 ms = gfn_to_memslot(kvm, cur_gfn);
2014 args->count = 0;
2015 args->start_gfn = cur_gfn;
2016 if (!ms)
2017 return 0;
2018 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2019 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2020
2021 while (args->count < bufsize) {
2022 hva = gfn_to_hva(kvm, cur_gfn);
2023 if (kvm_is_error_hva(hva))
2024 return 0;
2025 /* Decrement only if we actually flipped the bit to 0 */
2026 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2027 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2028 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2029 pgstev = 0;
2030 /* Save the value */
2031 res[args->count++] = (pgstev >> 24) & 0x43;
2032 /* If the next bit is too far away, stop. */
2033 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2034 return 0;
2035 /* If we reached the previous "next", find the next one */
2036 if (cur_gfn == next_gfn)
2037 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2038 /* Reached the end of memory or of the buffer, stop */
2039 if ((next_gfn >= mem_end) ||
2040 (next_gfn - args->start_gfn >= bufsize))
2041 return 0;
2042 cur_gfn++;
2043 /* Reached the end of the current memslot, take the next one. */
2044 if (cur_gfn - ms->base_gfn >= ms->npages) {
2045 ms = gfn_to_memslot(kvm, cur_gfn);
2046 if (!ms)
2047 return 0;
2048 }
2049 }
2050 return 0;
2051}
2052
4036e387
CI		 2053/*
2054 * This function searches for the next page with dirty CMMA attributes, and
2055 * saves the attributes in the buffer up to either the end of the buffer or
2056 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2057 * no trailing clean bytes are saved.
2058 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2059 * output buffer will indicate 0 as length.
2060 */
2061static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2062 struct kvm_s390_cmma_log *args)
2063{
afdad616
CI		 2064 unsigned long bufsize;
2065 int srcu_idx, peek, ret;
2066 u8 *values;
4036e387 2067
afdad616 2068 if (!kvm->arch.use_cmma)
4036e387
CI		 2069 return -ENXIO;
2070 /* Invalid/unsupported flags were specified */
2071 if (args->flags & ~KVM_S390_CMMA_PEEK)
2072 return -EINVAL;
2073 /* Migration mode query, and we are not doing a migration */
2074 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
afdad616 2075 if (!peek && !kvm->arch.migration_mode)
4036e387
CI		 2076 return -EINVAL;
2077 /* CMMA is disabled or was not used, or the buffer has length zero */
2078 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
c9f0a2b8 2079 if (!bufsize || !kvm->mm->context.uses_cmm) {
4036e387
CI		 2080 memset(args, 0, sizeof(*args));
2081 return 0;
2082 }
afdad616
CI		 2083 /* We are not peeking, and there are no dirty pages */
2084 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2085 memset(args, 0, sizeof(*args));
2086 return 0;
4036e387
CI		 2087 }
2088
afdad616
CI		 2089 values = vmalloc(bufsize);
2090 if (!values)
4036e387
CI		 2091 return -ENOMEM;
2092
4036e387
CI		 2093 down_read(&kvm->mm->mmap_sem);
2094 srcu_idx = srcu_read_lock(&kvm->srcu);
afdad616
CI		 2095 if (peek)
2096 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2097 else
2098 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
4036e387
CI		 2099 srcu_read_unlock(&kvm->srcu, srcu_idx);
2100 up_read(&kvm->mm->mmap_sem);
4036e387 2101
afdad616
CI		 2102 if (kvm->arch.migration_mode)
2103 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2104 else
2105 args->remaining = 0;
4036e387 2106
afdad616
CI		 2107 if (copy_to_user((void __user *)args->values, values, args->count))
2108 ret = -EFAULT;
2109
2110 vfree(values);
2111 return ret;
4036e387
CI		 2112}
2113
2114/*
2115 * This function sets the CMMA attributes for the given pages. If the input
2116 * buffer has zero length, no action is taken, otherwise the attributes are
c9f0a2b8 2117 * set and the mm->context.uses_cmm flag is set.
4036e387
CI		 2118 */
2119static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2120 const struct kvm_s390_cmma_log *args)
2121{
2122 unsigned long hva, mask, pgstev, i;
2123 uint8_t *bits;
2124 int srcu_idx, r = 0;
2125
2126 mask = args->mask;
2127
2128 if (!kvm->arch.use_cmma)
2129 return -ENXIO;
2130 /* invalid/unsupported flags */
2131 if (args->flags != 0)
2132 return -EINVAL;
2133 /* Enforce sane limit on memory allocation */
2134 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2135 return -EINVAL;
2136 /* Nothing to do */
2137 if (args->count == 0)
2138 return 0;
2139
42bc47b3 2140 bits = vmalloc(array_size(sizeof(*bits), args->count));
4036e387
CI		 2141 if (!bits)
2142 return -ENOMEM;
2143
2144 r = copy_from_user(bits, (void __user *)args->values, args->count);
2145 if (r) {
2146 r = -EFAULT;
2147 goto out;
2148 }
2149
2150 down_read(&kvm->mm->mmap_sem);
2151 srcu_idx = srcu_read_lock(&kvm->srcu);
2152 for (i = 0; i < args->count; i++) {
2153 hva = gfn_to_hva(kvm, args->start_gfn + i);
2154 if (kvm_is_error_hva(hva)) {
2155 r = -EFAULT;
2156 break;
2157 }
2158
2159 pgstev = bits[i];
2160 pgstev = pgstev << 24;
1bab1c02 2161 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
4036e387
CI		 2162 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2163 }
2164 srcu_read_unlock(&kvm->srcu, srcu_idx);
2165 up_read(&kvm->mm->mmap_sem);
2166
c9f0a2b8 2167 if (!kvm->mm->context.uses_cmm) {
4036e387 2168 down_write(&kvm->mm->mmap_sem);
c9f0a2b8 2169 kvm->mm->context.uses_cmm = 1;
4036e387
CI		 2170 up_write(&kvm->mm->mmap_sem);
2171 }
2172out:
2173 vfree(bits);
2174 return r;
2175}
2176
29b40f10
JF		 2177static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp)
2178{
2179 struct kvm_vcpu *vcpu;
2180 u16 rc, rrc;
2181 int ret = 0;
2182 int i;
2183
2184 /*
2185 * We ignore failures and try to destroy as many CPUs as possible.
2186 * At the same time we must not free the assigned resources when
2187 * this fails, as the ultravisor has still access to that memory.
2188 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2189 * behind.
2190 * We want to return the first failure rc and rrc, though.
2191 */
2192 kvm_for_each_vcpu(i, vcpu, kvm) {
2193 mutex_lock(&vcpu->mutex);
2194 if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
2195 *rcp = rc;
2196 *rrcp = rrc;
2197 ret = -EIO;
2198 }
2199 mutex_unlock(&vcpu->mutex);
2200 }
2201 return ret;
2202}
2203
2204static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2205{
2206 int i, r = 0;
2207 u16 dummy;
2208
2209 struct kvm_vcpu *vcpu;
2210
2211 kvm_for_each_vcpu(i, vcpu, kvm) {
2212 mutex_lock(&vcpu->mutex);
2213 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2214 mutex_unlock(&vcpu->mutex);
2215 if (r)
2216 break;
2217 }
2218 if (r)
2219 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2220 return r;
2221}
2222
2223static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2224{
2225 int r = 0;
2226 u16 dummy;
2227 void __user *argp = (void __user *)cmd->data;
2228
2229 switch (cmd->cmd) {
2230 case KVM_PV_ENABLE: {
2231 r = -EINVAL;
2232 if (kvm_s390_pv_is_protected(kvm))
2233 break;
2234
2235 /*
2236 * FMT 4 SIE needs esca. As we never switch back to bsca from
2237 * esca, we need no cleanup in the error cases below
2238 */
2239 r = sca_switch_to_extended(kvm);
2240 if (r)
2241 break;
2242
fa0c5eab
JF		 2243 down_write(&current->mm->mmap_sem);
2244 r = gmap_mark_unmergeable();
2245 up_write(&current->mm->mmap_sem);
2246 if (r)
2247 break;
2248
29b40f10
JF		 2249 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2250 if (r)
2251 break;
2252
2253 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2254 if (r)
2255 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
0890ddea
CB		 2256
2257 /* we need to block service interrupts from now on */
2258 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
29b40f10
JF		 2259 break;
2260 }
2261 case KVM_PV_DISABLE: {
2262 r = -EINVAL;
2263 if (!kvm_s390_pv_is_protected(kvm))
2264 break;
2265
2266 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2267 /*
2268 * If a CPU could not be destroyed, destroy VM will also fail.
2269 * There is no point in trying to destroy it. Instead return
2270 * the rc and rrc from the first CPU that failed destroying.
2271 */
2272 if (r)
2273 break;
2274 r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
0890ddea
CB		 2275
2276 /* no need to block service interrupts any more */
2277 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
29b40f10
JF		 2278 break;
2279 }
2280 case KVM_PV_SET_SEC_PARMS: {
2281 struct kvm_s390_pv_sec_parm parms = {};
2282 void *hdr;
2283
2284 r = -EINVAL;
2285 if (!kvm_s390_pv_is_protected(kvm))
2286 break;
2287
2288 r = -EFAULT;
2289 if (copy_from_user(&parms, argp, sizeof(parms)))
2290 break;
2291
2292 /* Currently restricted to 8KB */
2293 r = -EINVAL;
2294 if (parms.length > PAGE_SIZE * 2)
2295 break;
2296
2297 r = -ENOMEM;
2298 hdr = vmalloc(parms.length);
2299 if (!hdr)
2300 break;
2301
2302 r = -EFAULT;
2303 if (!copy_from_user(hdr, (void __user *)parms.origin,
2304 parms.length))
2305 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2306 &cmd->rc, &cmd->rrc);
2307
2308 vfree(hdr);
2309 break;
2310 }
2311 case KVM_PV_UNPACK: {
2312 struct kvm_s390_pv_unp unp = {};
2313
2314 r = -EINVAL;
2315 if (!kvm_s390_pv_is_protected(kvm))
2316 break;
2317
2318 r = -EFAULT;
2319 if (copy_from_user(&unp, argp, sizeof(unp)))
2320 break;
2321
2322 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2323 &cmd->rc, &cmd->rrc);
2324 break;
2325 }
2326 case KVM_PV_VERIFY: {
2327 r = -EINVAL;
2328 if (!kvm_s390_pv_is_protected(kvm))
2329 break;
2330
2331 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2332 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2333 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2334 cmd->rrc);
2335 break;
2336 }
e0d2773d
JF		 2337 case KVM_PV_PREP_RESET: {
2338 r = -EINVAL;
2339 if (!kvm_s390_pv_is_protected(kvm))
2340 break;
2341
2342 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2343 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2344 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2345 cmd->rc, cmd->rrc);
2346 break;
2347 }
2348 case KVM_PV_UNSHARE_ALL: {
2349 r = -EINVAL;
2350 if (!kvm_s390_pv_is_protected(kvm))
2351 break;
2352
2353 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2354 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2355 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2356 cmd->rc, cmd->rrc);
2357 break;
2358 }
29b40f10
JF		 2359 default:
2360 r = -ENOTTY;
2361 }
2362 return r;
2363}
2364
b0c632db
HC		 2365long kvm_arch_vm_ioctl(struct file *filp,
2366 unsigned int ioctl, unsigned long arg)
2367{
2368 struct kvm *kvm = filp->private_data;
2369 void __user *argp = (void __user *)arg;
f2061656 2370 struct kvm_device_attr attr;
b0c632db
HC		 2371 int r;
2372
2373 switch (ioctl) {
ba5c1e9b
CO		 2374 case KVM_S390_INTERRUPT: {
2375 struct kvm_s390_interrupt s390int;
2376
2377 r = -EFAULT;
2378 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2379 break;
2380 r = kvm_s390_inject_vm(kvm, &s390int);
2381 break;
2382 }
84223598
CH		 2383 case KVM_CREATE_IRQCHIP: {
2384 struct kvm_irq_routing_entry routing;
2385
2386 r = -EINVAL;
2387 if (kvm->arch.use_irqchip) {
2388 /* Set up dummy routing. */
2389 memset(&routing, 0, sizeof(routing));
152b2839 2390 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 2391 }
2392 break;
2393 }
f2061656
DD		 2394 case KVM_SET_DEVICE_ATTR: {
2395 r = -EFAULT;
2396 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2397 break;
2398 r = kvm_s390_vm_set_attr(kvm, &attr);
2399 break;
2400 }
2401 case KVM_GET_DEVICE_ATTR: {
2402 r = -EFAULT;
2403 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2404 break;
2405 r = kvm_s390_vm_get_attr(kvm, &attr);
2406 break;
2407 }
2408 case KVM_HAS_DEVICE_ATTR: {
2409 r = -EFAULT;
2410 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2411 break;
2412 r = kvm_s390_vm_has_attr(kvm, &attr);
2413 break;
2414 }
30ee2a98
JH		 2415 case KVM_S390_GET_SKEYS: {
2416 struct kvm_s390_skeys args;
2417
2418 r = -EFAULT;
2419 if (copy_from_user(&args, argp,
2420 sizeof(struct kvm_s390_skeys)))
2421 break;
2422 r = kvm_s390_get_skeys(kvm, &args);
2423 break;
2424 }
2425 case KVM_S390_SET_SKEYS: {
2426 struct kvm_s390_skeys args;
2427
2428 r = -EFAULT;
2429 if (copy_from_user(&args, argp,
2430 sizeof(struct kvm_s390_skeys)))
2431 break;
2432 r = kvm_s390_set_skeys(kvm, &args);
2433 break;
2434 }
4036e387
CI		 2435 case KVM_S390_GET_CMMA_BITS: {
2436 struct kvm_s390_cmma_log args;
2437
2438 r = -EFAULT;
2439 if (copy_from_user(&args, argp, sizeof(args)))
2440 break;
1de1ea7e 2441 mutex_lock(&kvm->slots_lock);
4036e387 2442 r = kvm_s390_get_cmma_bits(kvm, &args);
1de1ea7e 2443 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 2444 if (!r) {
2445 r = copy_to_user(argp, &args, sizeof(args));
2446 if (r)
2447 r = -EFAULT;
2448 }
2449 break;
2450 }
2451 case KVM_S390_SET_CMMA_BITS: {
2452 struct kvm_s390_cmma_log args;
2453
2454 r = -EFAULT;
2455 if (copy_from_user(&args, argp, sizeof(args)))
2456 break;
1de1ea7e 2457 mutex_lock(&kvm->slots_lock);
4036e387 2458 r = kvm_s390_set_cmma_bits(kvm, &args);
1de1ea7e 2459 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 2460 break;
2461 }
29b40f10
JF		 2462 case KVM_S390_PV_COMMAND: {
2463 struct kvm_pv_cmd args;
2464
fe28c786
JF		 2465 /* protvirt means user sigp */
2466 kvm->arch.user_cpu_state_ctrl = 1;
29b40f10
JF		 2467 r = 0;
2468 if (!is_prot_virt_host()) {
2469 r = -EINVAL;
2470 break;
2471 }
2472 if (copy_from_user(&args, argp, sizeof(args))) {
2473 r = -EFAULT;
2474 break;
2475 }
2476 if (args.flags) {
2477 r = -EINVAL;
2478 break;
2479 }
2480 mutex_lock(&kvm->lock);
2481 r = kvm_s390_handle_pv(kvm, &args);
2482 mutex_unlock(&kvm->lock);
2483 if (copy_to_user(argp, &args, sizeof(args))) {
2484 r = -EFAULT;
2485 break;
2486 }
2487 break;
2488 }
b0c632db 2489 default:
367e1319 2490 r = -ENOTTY;
b0c632db
HC		 2491 }
2492
2493 return r;
2494}
2495
45c9b47c
TK		 2496static int kvm_s390_apxa_installed(void)
2497{
e585b24a 2498 struct ap_config_info info;
45c9b47c 2499
e585b24a
TK		 2500 if (ap_instructions_available()) {
2501 if (ap_qci(&info) == 0)
2502 return info.apxa;
45c9b47c
TK		 2503 }
2504
2505 return 0;
2506}
2507
e585b24a
TK		 2508/*
2509 * The format of the crypto control block (CRYCB) is specified in the 3 low
2510 * order bits of the CRYCB designation (CRYCBD) field as follows:
2511 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2512 * AP extended addressing (APXA) facility are installed.
2513 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2514 * Format 2: Both the APXA and MSAX3 facilities are installed
2515 */
45c9b47c
TK		 2516static void kvm_s390_set_crycb_format(struct kvm *kvm)
2517{
2518 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2519
e585b24a
TK		 2520 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2521 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2522
2523 /* Check whether MSAX3 is installed */
2524 if (!test_kvm_facility(kvm, 76))
2525 return;
2526
45c9b47c
TK		 2527 if (kvm_s390_apxa_installed())
2528 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2529 else
2530 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2531}
2532
0e237e44
PM		 2533void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2534 unsigned long *aqm, unsigned long *adm)
2535{
2536 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2537
2538 mutex_lock(&kvm->lock);
2539 kvm_s390_vcpu_block_all(kvm);
2540
2541 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2542 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2543 memcpy(crycb->apcb1.apm, apm, 32);
2544 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2545 apm[0], apm[1], apm[2], apm[3]);
2546 memcpy(crycb->apcb1.aqm, aqm, 32);
2547 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2548 aqm[0], aqm[1], aqm[2], aqm[3]);
2549 memcpy(crycb->apcb1.adm, adm, 32);
2550 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2551 adm[0], adm[1], adm[2], adm[3]);
2552 break;
2553 case CRYCB_FORMAT1:
2554 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2555 memcpy(crycb->apcb0.apm, apm, 8);
2556 memcpy(crycb->apcb0.aqm, aqm, 2);
2557 memcpy(crycb->apcb0.adm, adm, 2);
2558 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2559 apm[0], *((unsigned short *)aqm),
2560 *((unsigned short *)adm));
2561 break;
2562 default: /* Can not happen */
2563 break;
2564 }
2565
2566 /* recreate the shadow crycb for each vcpu */
2567 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2568 kvm_s390_vcpu_unblock_all(kvm);
2569 mutex_unlock(&kvm->lock);
2570}
2571EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2572
42104598
TK		 2573void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2574{
2575 mutex_lock(&kvm->lock);
2576 kvm_s390_vcpu_block_all(kvm);
2577
2578 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2579 sizeof(kvm->arch.crypto.crycb->apcb0));
2580 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2581 sizeof(kvm->arch.crypto.crycb->apcb1));
2582
0e237e44 2583 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
6cc571b1
PM		 2584 /* recreate the shadow crycb for each vcpu */
2585 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
42104598
TK		 2586 kvm_s390_vcpu_unblock_all(kvm);
2587 mutex_unlock(&kvm->lock);
2588}
2589EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2590
9bb0ec09 2591static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 2592{
9bb0ec09
DH		 2593 struct cpuid cpuid;
2594
2595 get_cpu_id(&cpuid);
2596 cpuid.version = 0xff;
2597 return *((u64 *) &cpuid);
9d8d5786
MM		 2598}
2599
c54f0d6a 2600static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 2601{
c54f0d6a 2602 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 2603 kvm_s390_set_crycb_format(kvm);
5102ee87 2604
e585b24a
TK		 2605 if (!test_kvm_facility(kvm, 76))
2606 return;
2607
ed6f76b4
TK		 2608 /* Enable AES/DEA protected key functions by default */
2609 kvm->arch.crypto.aes_kw = 1;
2610 kvm->arch.crypto.dea_kw = 1;
2611 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2612 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2613 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2614 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 2615}
2616
7d43bafc
ED		 2617static void sca_dispose(struct kvm *kvm)
2618{
2619 if (kvm->arch.use_esca)
5e044315 2620 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 2621 else
2622 free_page((unsigned long)(kvm->arch.sca));
2623 kvm->arch.sca = NULL;
2624}
2625
e08b9637 2626int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 2627{
76a6dd72 2628 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 2629 int i, rc;
b0c632db 2630 char debug_name[16];
f6c137ff 2631 static unsigned long sca_offset;
b0c632db 2632
e08b9637
CO		 2633 rc = -EINVAL;
2634#ifdef CONFIG_KVM_S390_UCONTROL
2635 if (type & ~KVM_VM_S390_UCONTROL)
2636 goto out_err;
2637 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2638 goto out_err;
2639#else
2640 if (type)
2641 goto out_err;
2642#endif
2643
b0c632db
HC		 2644 rc = s390_enable_sie();
2645 if (rc)
d89f5eff 2646 goto out_err;
b0c632db 2647
b290411a
CO		 2648 rc = -ENOMEM;
2649
76a6dd72
DH		 2650 if (!sclp.has_64bscao)
2651 alloc_flags |= GFP_DMA;
5e044315 2652 rwlock_init(&kvm->arch.sca_lock);
9ac96d75 2653 /* start with basic SCA */
76a6dd72 2654 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 2655 if (!kvm->arch.sca)
d89f5eff 2656 goto out_err;
0d9ce162 2657 mutex_lock(&kvm_lock);
c5c2c393 2658 sca_offset += 16;
bc784cce 2659 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 2660 sca_offset = 0;
bc784cce
ED		 2661 kvm->arch.sca = (struct bsca_block *)
2662 ((char *) kvm->arch.sca + sca_offset);
0d9ce162 2663 mutex_unlock(&kvm_lock);
b0c632db
HC		 2664
2665 sprintf(debug_name, "kvm-%u", current->pid);
2666
1cb9cf72 2667 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 2668 if (!kvm->arch.dbf)
40f5b735 2669 goto out_err;
b0c632db 2670
19114beb 2671 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
c54f0d6a
DH		 2672 kvm->arch.sie_page2 =
2673 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2674 if (!kvm->arch.sie_page2)
40f5b735 2675 goto out_err;
9d8d5786 2676
25c84dba 2677 kvm->arch.sie_page2->kvm = kvm;
c54f0d6a 2678 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
c3b9e3e1
CB		 2679
2680 for (i = 0; i < kvm_s390_fac_size(); i++) {
2681 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2682 (kvm_s390_fac_base[i] |
2683 kvm_s390_fac_ext[i]);
2684 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2685 kvm_s390_fac_base[i];
2686 }
346fa2f8 2687 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
981467c9 2688
1935222d
DH		 2689 /* we are always in czam mode - even on pre z14 machines */
2690 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2691 set_kvm_facility(kvm->arch.model.fac_list, 138);
2692 /* we emulate STHYI in kvm */
95ca2cb5
JF		 2693 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2694 set_kvm_facility(kvm->arch.model.fac_list, 74);
1bab1c02
CI		 2695 if (MACHINE_HAS_TLB_GUEST) {
2696 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2697 set_kvm_facility(kvm->arch.model.fac_list, 147);
2698 }
95ca2cb5 2699
05f31e3b
PM		 2700 if (css_general_characteristics.aiv && test_facility(65))
2701 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2702
9bb0ec09 2703 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 2704 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 2705
c54f0d6a 2706 kvm_s390_crypto_init(kvm);
5102ee87 2707
51978393 2708 mutex_init(&kvm->arch.float_int.ais_lock);
ba5c1e9b 2709 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 2710 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2711 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 2712 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 2713 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 2714
b0c632db 2715 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 2716 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 2717
e08b9637
CO		 2718 if (type & KVM_VM_S390_UCONTROL) {
2719 kvm->arch.gmap = NULL;
a3a92c31 2720 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 2721 } else {
32e6b236 2722 if (sclp.hamax == U64_MAX)
ee71d16d 2723 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 2724 else
ee71d16d 2725 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 2726 sclp.hamax + 1);
6ea427bb 2727 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 2728 if (!kvm->arch.gmap)
40f5b735 2729 goto out_err;
2c70fe44 2730 kvm->arch.gmap->private = kvm;
24eb3a82 2731 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 2732 }
fa6b7fe9 2733
c9f0a2b8 2734 kvm->arch.use_pfmfi = sclp.has_pfmfi;
55531b74 2735 kvm->arch.use_skf = sclp.has_skey;
8ad35755 2736 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 2737 kvm_s390_vsie_init(kvm);
cc674ef2
MM		 2738 if (use_gisa)
2739 kvm_s390_gisa_init(kvm);
8335713a 2740 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 2741
d89f5eff 2742 return 0;
40f5b735 2743out_err:
c54f0d6a 2744 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 2745 debug_unregister(kvm->arch.dbf);
7d43bafc 2746 sca_dispose(kvm);
78f26131 2747 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 2748 return rc;
b0c632db
HC		 2749}
2750
d329c035
CB		 2751void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2752{
29b40f10
JF		 2753 u16 rc, rrc;
2754
d329c035 2755 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 2756 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 2757 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 2758 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 2759 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 2760 sca_del_vcpu(vcpu);
27e0393f
CO		 2761
2762 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 2763 gmap_remove(vcpu->arch.gmap);
27e0393f 2764
e6db1d61 2765 if (vcpu->kvm->arch.use_cmma)
b31605c1 2766 kvm_s390_vcpu_unsetup_cmma(vcpu);
29b40f10
JF		 2767 /* We can not hold the vcpu mutex here, we are already dying */
2768 if (kvm_s390_pv_cpu_get_handle(vcpu))
2769 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
d329c035 2770 free_page((unsigned long)(vcpu->arch.sie_block));
d329c035
CB		 2771}
2772
2773static void kvm_free_vcpus(struct kvm *kvm)
2774{
2775 unsigned int i;
988a2cae 2776 struct kvm_vcpu *vcpu;
d329c035 2777
988a2cae 2778 kvm_for_each_vcpu(i, vcpu, kvm)
4543bdc0 2779 kvm_vcpu_destroy(vcpu);
988a2cae
GN		 2780
2781 mutex_lock(&kvm->lock);
2782 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2783 kvm->vcpus[i] = NULL;
2784
2785 atomic_set(&kvm->online_vcpus, 0);
2786 mutex_unlock(&kvm->lock);
d329c035
CB		 2787}
2788
b0c632db
HC		 2789void kvm_arch_destroy_vm(struct kvm *kvm)
2790{
29b40f10
JF		 2791 u16 rc, rrc;
2792
d329c035 2793 kvm_free_vcpus(kvm);
7d43bafc 2794 sca_dispose(kvm);
d7c5cb01 2795 kvm_s390_gisa_destroy(kvm);
29b40f10
JF		 2796 /*
2797 * We are already at the end of life and kvm->lock is not taken.
2798 * This is ok as the file descriptor is closed by now and nobody
2799 * can mess with the pv state. To avoid lockdep_assert_held from
2800 * complaining we do not use kvm_s390_pv_is_protected.
2801 */
2802 if (kvm_s390_pv_get_handle(kvm))
2803 kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
2804 debug_unregister(kvm->arch.dbf);
c54f0d6a 2805 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 2806 if (!kvm_is_ucontrol(kvm))
6ea427bb 2807 gmap_remove(kvm->arch.gmap);
841b91c5 2808 kvm_s390_destroy_adapters(kvm);
67335e63 2809 kvm_s390_clear_float_irqs(kvm);
a3508fbe 2810 kvm_s390_vsie_destroy(kvm);
8335713a 2811 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 2812}
2813
2814/* Section: vcpu related */
dafd032a
DD		 2815static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2816{
6ea427bb 2817 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 2818 if (!vcpu->arch.gmap)
2819 return -ENOMEM;
2820 vcpu->arch.gmap->private = vcpu->kvm;
2821
2822 return 0;
2823}
2824
a6e2f683
ED		 2825static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2826{
a6940674
DH		 2827 if (!kvm_s390_use_sca_entries())
2828 return;
5e044315 2829 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 2830 if (vcpu->kvm->arch.use_esca) {
2831 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2832
7d43bafc 2833 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 2834 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 2835 } else {
2836 struct bsca_block *sca = vcpu->kvm->arch.sca;
2837
2838 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 2839 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 2840 }
5e044315 2841 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 2842}
2843
eaa78f34 2844static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 2845{
a6940674
DH		 2846 if (!kvm_s390_use_sca_entries()) {
2847 struct bsca_block *sca = vcpu->kvm->arch.sca;
2848
2849 /* we still need the basic sca for the ipte control */
2850 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2851 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
f07afa04 2852 return;
a6940674 2853 }
eaa78f34
DH		 2854 read_lock(&vcpu->kvm->arch.sca_lock);
2855 if (vcpu->kvm->arch.use_esca) {
2856 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 2857
eaa78f34 2858 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2859 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2860 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 2861 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 2862 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 2863 } else {
eaa78f34 2864 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2865
eaa78f34 2866 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2867 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2868 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 2869 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 2870 }
eaa78f34 2871 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 2872}
2873
2874/* Basic SCA to Extended SCA data copy routines */
2875static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2876{
2877 d->sda = s->sda;
2878 d->sigp_ctrl.c = s->sigp_ctrl.c;
2879 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2880}
2881
2882static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2883{
2884 int i;
2885
2886 d->ipte_control = s->ipte_control;
2887 d->mcn[0] = s->mcn;
2888 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2889 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2890}
2891
2892static int sca_switch_to_extended(struct kvm *kvm)
2893{
2894 struct bsca_block *old_sca = kvm->arch.sca;
2895 struct esca_block *new_sca;
2896 struct kvm_vcpu *vcpu;
2897 unsigned int vcpu_idx;
2898 u32 scaol, scaoh;
2899
29b40f10
JF		 2900 if (kvm->arch.use_esca)
2901 return 0;
2902
5e044315
ED		 2903 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2904 if (!new_sca)
2905 return -ENOMEM;
2906
2907 scaoh = (u32)((u64)(new_sca) >> 32);
2908 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2909
2910 kvm_s390_vcpu_block_all(kvm);
2911 write_lock(&kvm->arch.sca_lock);
2912
2913 sca_copy_b_to_e(new_sca, old_sca);
2914
2915 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2916 vcpu->arch.sie_block->scaoh = scaoh;
2917 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 2918 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 2919 }
2920 kvm->arch.sca = new_sca;
2921 kvm->arch.use_esca = 1;
2922
2923 write_unlock(&kvm->arch.sca_lock);
2924 kvm_s390_vcpu_unblock_all(kvm);
2925
2926 free_page((unsigned long)old_sca);
2927
8335713a
CB		 2928 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2929 old_sca, kvm->arch.sca);
5e044315 2930 return 0;
a6e2f683
ED		 2931}
2932
2933static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2934{
5e044315
ED		 2935 int rc;
2936
a6940674
DH		 2937 if (!kvm_s390_use_sca_entries()) {
2938 if (id < KVM_MAX_VCPUS)
2939 return true;
2940 return false;
2941 }
5e044315
ED		 2942 if (id < KVM_S390_BSCA_CPU_SLOTS)
2943 return true;
76a6dd72 2944 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 2945 return false;
2946
2947 mutex_lock(&kvm->lock);
2948 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2949 mutex_unlock(&kvm->lock);
2950
2951 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 2952}
2953
db0758b2
DH		 2954/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2955static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2956{
2957 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 2958 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 2959 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 2960 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2961}
2962
2963/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2964static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2965{
2966 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 2967 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2968 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2969 vcpu->arch.cputm_start = 0;
9c23a131 2970 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2971}
2972
2973/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2974static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2975{
2976 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2977 vcpu->arch.cputm_enabled = true;
2978 __start_cpu_timer_accounting(vcpu);
2979}
2980
2981/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2982static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2983{
2984 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2985 __stop_cpu_timer_accounting(vcpu);
2986 vcpu->arch.cputm_enabled = false;
2987}
2988
2989static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2990{
2991 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2992 __enable_cpu_timer_accounting(vcpu);
2993 preempt_enable();
2994}
2995
2996static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2997{
2998 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2999 __disable_cpu_timer_accounting(vcpu);
3000 preempt_enable();
3001}
3002
4287f247
DH		 3003/* set the cpu timer - may only be called from the VCPU thread itself */
3004void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3005{
db0758b2 3006 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 3007 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 3008 if (vcpu->arch.cputm_enabled)
3009 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 3010 vcpu->arch.sie_block->cputm = cputm;
9c23a131 3011 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 3012 preempt_enable();
4287f247
DH		 3013}
3014
db0758b2 3015/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 3016__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3017{
9c23a131 3018 unsigned int seq;
db0758b2 3019 __u64 value;
db0758b2
DH		 3020
3021 if (unlikely(!vcpu->arch.cputm_enabled))
3022 return vcpu->arch.sie_block->cputm;
3023
9c23a131
DH		 3024 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3025 do {
3026 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3027 /*
3028 * If the writer would ever execute a read in the critical
3029 * section, e.g. in irq context, we have a deadlock.
3030 */
3031 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3032 value = vcpu->arch.sie_block->cputm;
3033 /* if cputm_start is 0, accounting is being started/stopped */
3034 if (likely(vcpu->arch.cputm_start))
3035 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3036 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3037 preempt_enable();
db0758b2 3038 return value;
4287f247
DH		 3039}
3040
b0c632db
HC		 3041void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3042{
9977e886 3043
37d9df98 3044 gmap_enable(vcpu->arch.enabled_gmap);
ef8f4f49 3045 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
5ebda316 3046 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 3047 __start_cpu_timer_accounting(vcpu);
01a745ac 3048 vcpu->cpu = cpu;
b0c632db
HC		 3049}
3050
3051void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3052{
01a745ac 3053 vcpu->cpu = -1;
5ebda316 3054 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 3055 __stop_cpu_timer_accounting(vcpu);
9daecfc6 3056 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
37d9df98
DH		 3057 vcpu->arch.enabled_gmap = gmap_get_enabled();
3058 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 3059
b0c632db
HC		 3060}
3061
31928aa5 3062void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 3063{
72f25020 3064 mutex_lock(&vcpu->kvm->lock);
fdf03650 3065 preempt_disable();
72f25020 3066 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
d16b52cb 3067 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
fdf03650 3068 preempt_enable();
72f25020 3069 mutex_unlock(&vcpu->kvm->lock);
25508824 3070 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 3071 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 3072 sca_add_vcpu(vcpu);
25508824 3073 }
6502a34c
DH		 3074 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3075 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 3076 /* make vcpu_load load the right gmap on the first trigger */
3077 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 3078}
3079
8ec2fa52
CB		 3080static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3081{
3082 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3083 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3084 return true;
3085 return false;
3086}
3087
3088static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3089{
3090 /* At least one ECC subfunction must be present */
3091 return kvm_has_pckmo_subfunc(kvm, 32) ||
3092 kvm_has_pckmo_subfunc(kvm, 33) ||
3093 kvm_has_pckmo_subfunc(kvm, 34) ||
3094 kvm_has_pckmo_subfunc(kvm, 40) ||
3095 kvm_has_pckmo_subfunc(kvm, 41);
3096
3097}
3098
5102ee87
TK		 3099static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3100{
e585b24a
TK		 3101 /*
3102 * If the AP instructions are not being interpreted and the MSAX3
3103 * facility is not configured for the guest, there is nothing to set up.
3104 */
3105 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 3106 return;
3107
e585b24a 3108 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
a374e892 3109 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
37940fb0 3110 vcpu->arch.sie_block->eca &= ~ECA_APIE;
8ec2fa52 3111 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
a374e892 3112
e585b24a
TK		 3113 if (vcpu->kvm->arch.crypto.apie)
3114 vcpu->arch.sie_block->eca |= ECA_APIE;
a374e892 3115
e585b24a 3116 /* Set up protected key support */
8ec2fa52 3117 if (vcpu->kvm->arch.crypto.aes_kw) {
a374e892 3118 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
8ec2fa52
CB		 3119 /* ecc is also wrapped with AES key */
3120 if (kvm_has_pckmo_ecc(vcpu->kvm))
3121 vcpu->arch.sie_block->ecd |= ECD_ECC;
3122 }
3123
a374e892
TK		 3124 if (vcpu->kvm->arch.crypto.dea_kw)
3125 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
5102ee87
TK		 3126}
3127
b31605c1
DD		 3128void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3129{
3130 free_page(vcpu->arch.sie_block->cbrlo);
3131 vcpu->arch.sie_block->cbrlo = 0;
3132}
3133
3134int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3135{
3136 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
3137 if (!vcpu->arch.sie_block->cbrlo)
3138 return -ENOMEM;
b31605c1
DD		 3139 return 0;
3140}
3141
91520f1a
MM		 3142static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3143{
3144 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3145
91520f1a 3146 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 3147 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 3148 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 3149}
3150
ff72bb55 3151static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
b0c632db 3152{
b31605c1 3153 int rc = 0;
29b40f10 3154 u16 uvrc, uvrrc;
b31288fa 3155
9e6dabef
CH		 3156 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3157 CPUSTAT_SM |
a4a4f191
GH		 3158 CPUSTAT_STOPPED);
3159
53df84f8 3160 if (test_kvm_facility(vcpu->kvm, 78))
ef8f4f49 3161 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
53df84f8 3162 else if (test_kvm_facility(vcpu->kvm, 8))
ef8f4f49 3163 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
a4a4f191 3164
91520f1a
MM		 3165 kvm_s390_vcpu_setup_model(vcpu);
3166
bdab09f3
DH		 3167 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
3168 if (MACHINE_HAS_ESOP)
0c9d8683 3169 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 3170 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 3171 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 3172 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 3173 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 3174
c9f0a2b8 3175 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
0c9d8683 3176 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 3177 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 3178 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3179 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 3180 if (sclp.has_cei)
0c9d8683 3181 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 3182 if (sclp.has_ib)
0c9d8683 3183 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 3184 if (sclp.has_siif)
0c9d8683 3185 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 3186 if (sclp.has_sigpif)
0c9d8683 3187 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 3188 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 3189 vcpu->arch.sie_block->eca |= ECA_VX;
3190 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 3191 }
8fa1696e
CW		 3192 if (test_kvm_facility(vcpu->kvm, 139))
3193 vcpu->arch.sie_block->ecd |= ECD_MEF;
a3da7b4a
CB		 3194 if (test_kvm_facility(vcpu->kvm, 156))
3195 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
d7c5cb01
MM		 3196 if (vcpu->arch.sie_block->gd) {
3197 vcpu->arch.sie_block->eca |= ECA_AIV;
3198 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3199 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3200 }
4e0b1ab7
FZ		 3201 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
3202 | SDNXC;
c6e5f166 3203 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 3204
3205 if (sclp.has_kss)
ef8f4f49 3206 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
730cd632
FA		 3207 else
3208 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 3209
e6db1d61 3210 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 3211 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3212 if (rc)
3213 return rc;
b31288fa 3214 }
0ac96caf 3215 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 3216 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 3217
67d49d52
CW		 3218 vcpu->arch.sie_block->hpid = HPID_KVM;
3219
5102ee87
TK		 3220 kvm_s390_vcpu_crypto_setup(vcpu);
3221
29b40f10
JF		 3222 mutex_lock(&vcpu->kvm->lock);
3223 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3224 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3225 if (rc)
3226 kvm_s390_vcpu_unsetup_cmma(vcpu);
3227 }
3228 mutex_unlock(&vcpu->kvm->lock);
3229
b31605c1 3230 return rc;
b0c632db
HC		 3231}
3232
897cc38e
SC		 3233int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3234{
3235 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3236 return -EINVAL;
3237 return 0;
3238}
3239
e529ef66 3240int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
b0c632db 3241{
7feb6bb8 3242 struct sie_page *sie_page;
897cc38e 3243 int rc;
4d47555a 3244
da72ca4d 3245 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
7feb6bb8
MM		 3246 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
3247 if (!sie_page)
e529ef66 3248 return -ENOMEM;
b0c632db 3249
7feb6bb8
MM		 3250 vcpu->arch.sie_block = &sie_page->sie_block;
3251 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3252
efed1104
DH		 3253 /* the real guest size will always be smaller than msl */
3254 vcpu->arch.sie_block->mso = 0;
3255 vcpu->arch.sie_block->msl = sclp.hamax;
3256
e529ef66 3257 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
ba5c1e9b 3258 spin_lock_init(&vcpu->arch.local_int.lock);
e529ef66 3259 vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
4b9f9525
MM		 3260 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3261 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
9c23a131 3262 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 3263
321f8ee5
SC		 3264 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3265 kvm_clear_async_pf_completion_queue(vcpu);
3266 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3267 KVM_SYNC_GPRS |
3268 KVM_SYNC_ACRS |
3269 KVM_SYNC_CRS |
3270 KVM_SYNC_ARCH0 |
3271 KVM_SYNC_PFAULT;
3272 kvm_s390_set_prefix(vcpu, 0);
3273 if (test_kvm_facility(vcpu->kvm, 64))
3274 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3275 if (test_kvm_facility(vcpu->kvm, 82))
3276 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3277 if (test_kvm_facility(vcpu->kvm, 133))
3278 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3279 if (test_kvm_facility(vcpu->kvm, 156))
3280 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3281 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3282 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3283 */
3284 if (MACHINE_HAS_VX)
3285 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3286 else
3287 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3288
3289 if (kvm_is_ucontrol(vcpu->kvm)) {
3290 rc = __kvm_ucontrol_vcpu_init(vcpu);
3291 if (rc)
a2017f17 3292 goto out_free_sie_block;
321f8ee5
SC		 3293 }
3294
e529ef66
SC		 3295 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3296 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3297 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3298
ff72bb55
SC		 3299 rc = kvm_s390_vcpu_setup(vcpu);
3300 if (rc)
3301 goto out_ucontrol_uninit;
e529ef66 3302 return 0;
b0c632db 3303
ff72bb55
SC		 3304out_ucontrol_uninit:
3305 if (kvm_is_ucontrol(vcpu->kvm))
3306 gmap_remove(vcpu->arch.gmap);
7b06bf2f
WY		 3307out_free_sie_block:
3308 free_page((unsigned long)(vcpu->arch.sie_block));
e529ef66 3309 return rc;
b0c632db
HC		 3310}
3311
b0c632db
HC		 3312int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3313{
9a022067 3314 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 3315}
3316
199b5763
LM		 3317bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3318{
0546c63d 3319 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
199b5763
LM		 3320}
3321
27406cd5 3322void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 3323{
805de8f4 3324 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 3325 exit_sie(vcpu);
49b99e1e
CB		 3326}
3327
27406cd5 3328void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 3329{
805de8f4 3330 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 3331}
3332
8e236546
CB		 3333static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3334{
805de8f4 3335 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 3336 exit_sie(vcpu);
8e236546
CB		 3337}
3338
9ea59728
DH		 3339bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3340{
3341 return atomic_read(&vcpu->arch.sie_block->prog20) &
3342 (PROG_BLOCK_SIE | PROG_REQUEST);
3343}
3344
8e236546
CB		 3345static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3346{
9bf9fde2 3347 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 3348}
3349
49b99e1e 3350/*
9ea59728 3351 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
49b99e1e
CB		 3352 * If the CPU is not running (e.g. waiting as idle) the function will
3353 * return immediately. */
3354void exit_sie(struct kvm_vcpu *vcpu)
3355{
ef8f4f49 3356 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
9ea59728 3357 kvm_s390_vsie_kick(vcpu);
49b99e1e
CB		 3358 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3359 cpu_relax();
3360}
3361
8e236546
CB		 3362/* Kick a guest cpu out of SIE to process a request synchronously */
3363void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 3364{
8e236546
CB		 3365 kvm_make_request(req, vcpu);
3366 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 3367}
3368
414d3b07
MS		 3369static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3370 unsigned long end)
2c70fe44 3371{
2c70fe44
CB		 3372 struct kvm *kvm = gmap->private;
3373 struct kvm_vcpu *vcpu;
414d3b07
MS		 3374 unsigned long prefix;
3375 int i;
2c70fe44 3376
65d0b0d4
DH		 3377 if (gmap_is_shadow(gmap))
3378 return;
414d3b07
MS		 3379 if (start >= 1UL << 31)
3380 /* We are only interested in prefix pages */
3381 return;
2c70fe44
CB		 3382 kvm_for_each_vcpu(i, vcpu, kvm) {
3383 /* match against both prefix pages */
414d3b07
MS		 3384 prefix = kvm_s390_get_prefix(vcpu);
3385 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3386 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3387 start, end);
8e236546 3388 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 3389 }
3390 }
3391}
3392
8b905d28
CB		 3393bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3394{
3395 /* do not poll with more than halt_poll_max_steal percent of steal time */
3396 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3397 halt_poll_max_steal) {
3398 vcpu->stat.halt_no_poll_steal++;
3399 return true;
3400 }
3401 return false;
3402}
3403
b6d33834
CD		 3404int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3405{
3406 /* kvm common code refers to this, but never calls it */
3407 BUG();
3408 return 0;
3409}
3410
14eebd91
CO		 3411static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3412 struct kvm_one_reg *reg)
3413{
3414 int r = -EINVAL;
3415
3416 switch (reg->id) {
29b7c71b
CO		 3417 case KVM_REG_S390_TODPR:
3418 r = put_user(vcpu->arch.sie_block->todpr,
3419 (u32 __user *)reg->addr);
3420 break;
3421 case KVM_REG_S390_EPOCHDIFF:
3422 r = put_user(vcpu->arch.sie_block->epoch,
3423 (u64 __user *)reg->addr);
3424 break;
46a6dd1c 3425 case KVM_REG_S390_CPU_TIMER:
4287f247 3426 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 3427 (u64 __user *)reg->addr);
3428 break;
3429 case KVM_REG_S390_CLOCK_COMP:
3430 r = put_user(vcpu->arch.sie_block->ckc,
3431 (u64 __user *)reg->addr);
3432 break;
536336c2
DD		 3433 case KVM_REG_S390_PFTOKEN:
3434 r = put_user(vcpu->arch.pfault_token,
3435 (u64 __user *)reg->addr);
3436 break;
3437 case KVM_REG_S390_PFCOMPARE:
3438 r = put_user(vcpu->arch.pfault_compare,
3439 (u64 __user *)reg->addr);
3440 break;
3441 case KVM_REG_S390_PFSELECT:
3442 r = put_user(vcpu->arch.pfault_select,
3443 (u64 __user *)reg->addr);
3444 break;
672550fb
CB		 3445 case KVM_REG_S390_PP:
3446 r = put_user(vcpu->arch.sie_block->pp,
3447 (u64 __user *)reg->addr);
3448 break;
afa45ff5
CB		 3449 case KVM_REG_S390_GBEA:
3450 r = put_user(vcpu->arch.sie_block->gbea,
3451 (u64 __user *)reg->addr);
3452 break;
14eebd91
CO		 3453 default:
3454 break;
3455 }
3456
3457 return r;
3458}
3459
3460static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3461 struct kvm_one_reg *reg)
3462{
3463 int r = -EINVAL;
4287f247 3464 __u64 val;
14eebd91
CO		 3465
3466 switch (reg->id) {
29b7c71b
CO		 3467 case KVM_REG_S390_TODPR:
3468 r = get_user(vcpu->arch.sie_block->todpr,
3469 (u32 __user *)reg->addr);
3470 break;
3471 case KVM_REG_S390_EPOCHDIFF:
3472 r = get_user(vcpu->arch.sie_block->epoch,
3473 (u64 __user *)reg->addr);
3474 break;
46a6dd1c 3475 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 3476 r = get_user(val, (u64 __user *)reg->addr);
3477 if (!r)
3478 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 3479 break;
3480 case KVM_REG_S390_CLOCK_COMP:
3481 r = get_user(vcpu->arch.sie_block->ckc,
3482 (u64 __user *)reg->addr);
3483 break;
536336c2
DD		 3484 case KVM_REG_S390_PFTOKEN:
3485 r = get_user(vcpu->arch.pfault_token,
3486 (u64 __user *)reg->addr);
9fbd8082
DH		 3487 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3488 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 3489 break;
3490 case KVM_REG_S390_PFCOMPARE:
3491 r = get_user(vcpu->arch.pfault_compare,
3492 (u64 __user *)reg->addr);
3493 break;
3494 case KVM_REG_S390_PFSELECT:
3495 r = get_user(vcpu->arch.pfault_select,
3496 (u64 __user *)reg->addr);
3497 break;
672550fb
CB		 3498 case KVM_REG_S390_PP:
3499 r = get_user(vcpu->arch.sie_block->pp,
3500 (u64 __user *)reg->addr);
3501 break;
afa45ff5
CB		 3502 case KVM_REG_S390_GBEA:
3503 r = get_user(vcpu->arch.sie_block->gbea,
3504 (u64 __user *)reg->addr);
3505 break;
14eebd91
CO		 3506 default:
3507 break;
3508 }
3509
3510 return r;
3511}
b6d33834 3512
7de3f142 3513static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
b0c632db 3514{
7de3f142
JF		 3515 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3516 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3517 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3518
3519 kvm_clear_async_pf_completion_queue(vcpu);
3520 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3521 kvm_s390_vcpu_stop(vcpu);
3522 kvm_s390_clear_local_irqs(vcpu);
3523}
3524
3525static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3526{
3527 /* Initial reset is a superset of the normal reset */
3528 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3529
e93fc7b4
CB		 3530 /*
3531 * This equals initial cpu reset in pop, but we don't switch to ESA.
3532 * We do not only reset the internal data, but also ...
3533 */
7de3f142
JF		 3534 vcpu->arch.sie_block->gpsw.mask = 0;
3535 vcpu->arch.sie_block->gpsw.addr = 0;
3536 kvm_s390_set_prefix(vcpu, 0);
3537 kvm_s390_set_cpu_timer(vcpu, 0);
3538 vcpu->arch.sie_block->ckc = 0;
7de3f142
JF		 3539 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3540 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3541 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
e93fc7b4
CB		 3542
3543 /* ... the data in sync regs */
3544 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
3545 vcpu->run->s.regs.ckc = 0;
3546 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
3547 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
3548 vcpu->run->psw_addr = 0;
3549 vcpu->run->psw_mask = 0;
3550 vcpu->run->s.regs.todpr = 0;
3551 vcpu->run->s.regs.cputm = 0;
3552 vcpu->run->s.regs.ckc = 0;
3553 vcpu->run->s.regs.pp = 0;
3554 vcpu->run->s.regs.gbea = 1;
7de3f142 3555 vcpu->run->s.regs.fpc = 0;
0f303504
JF		 3556 /*
3557 * Do not reset these registers in the protected case, as some of
3558 * them are overlayed and they are not accessible in this case
3559 * anyway.
3560 */
3561 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3562 vcpu->arch.sie_block->gbea = 1;
3563 vcpu->arch.sie_block->pp = 0;
3564 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3565 vcpu->arch.sie_block->todpr = 0;
3566 }
7de3f142
JF		 3567}
3568
3569static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3570{
3571 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3572
3573 /* Clear reset is a superset of the initial reset */
3574 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3575
3576 memset(&regs->gprs, 0, sizeof(regs->gprs));
3577 memset(&regs->vrs, 0, sizeof(regs->vrs));
3578 memset(&regs->acrs, 0, sizeof(regs->acrs));
3579 memset(&regs->gscb, 0, sizeof(regs->gscb));
3580
3581 regs->etoken = 0;
3582 regs->etoken_extension = 0;
b0c632db
HC		 3583}
3584
3585int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3586{
875656fe 3587 vcpu_load(vcpu);
5a32c1af 3588 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
875656fe 3589 vcpu_put(vcpu);
b0c632db
HC		 3590 return 0;
3591}
3592
3593int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3594{
1fc9b76b 3595 vcpu_load(vcpu);
5a32c1af 3596 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1fc9b76b 3597 vcpu_put(vcpu);
b0c632db
HC		 3598 return 0;
3599}
3600
3601int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3602 struct kvm_sregs *sregs)
3603{
b4ef9d4e
CD		 3604 vcpu_load(vcpu);
3605
59674c1a 3606 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 3607 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b4ef9d4e
CD		 3608
3609 vcpu_put(vcpu);
b0c632db
HC		 3610 return 0;
3611}
3612
3613int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3614 struct kvm_sregs *sregs)
3615{
bcdec41c
CD		 3616 vcpu_load(vcpu);
3617
59674c1a 3618 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 3619 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
bcdec41c
CD		 3620
3621 vcpu_put(vcpu);
b0c632db
HC		 3622 return 0;
3623}
3624
3625int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3626{
6a96bc7f
CD		 3627 int ret = 0;
3628
3629 vcpu_load(vcpu);
3630
3631 if (test_fp_ctl(fpu->fpc)) {
3632 ret = -EINVAL;
3633 goto out;
3634 }
e1788bb9 3635 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 3636 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 3637 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3638 (freg_t *) fpu->fprs);
9abc2a08 3639 else
a7d4b8f2 3640 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
6a96bc7f
CD		 3641
3642out:
3643 vcpu_put(vcpu);
3644 return ret;
b0c632db
HC		 3645}
3646
3647int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3648{
1393123e
CD		 3649 vcpu_load(vcpu);
3650
9abc2a08
DH		 3651 /* make sure we have the latest values */
3652 save_fpu_regs();
3653 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 3654 convert_vx_to_fp((freg_t *) fpu->fprs,
3655 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 3656 else
a7d4b8f2 3657 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 3658 fpu->fpc = vcpu->run->s.regs.fpc;
1393123e
CD		 3659
3660 vcpu_put(vcpu);
b0c632db
HC		 3661 return 0;
3662}
3663
3664static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3665{
3666 int rc = 0;
3667
7a42fdc2 3668 if (!is_vcpu_stopped(vcpu))
b0c632db 3669 rc = -EBUSY;
d7b0b5eb
CO		 3670 else {
3671 vcpu->run->psw_mask = psw.mask;
3672 vcpu->run->psw_addr = psw.addr;
3673 }
b0c632db
HC		 3674 return rc;
3675}
3676
3677int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3678 struct kvm_translation *tr)
3679{
3680 return -EINVAL; /* not implemented yet */
3681}
3682
27291e21
DH		 3683#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3684 KVM_GUESTDBG_USE_HW_BP | \
3685 KVM_GUESTDBG_ENABLE)
3686
d0bfb940
JK		 3687int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3688 struct kvm_guest_debug *dbg)
b0c632db 3689{
27291e21
DH		 3690 int rc = 0;
3691
66b56562
CD		 3692 vcpu_load(vcpu);
3693
27291e21
DH		 3694 vcpu->guest_debug = 0;
3695 kvm_s390_clear_bp_data(vcpu);
3696
66b56562
CD		 3697 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3698 rc = -EINVAL;
3699 goto out;
3700 }
3701 if (!sclp.has_gpere) {
3702 rc = -EINVAL;
3703 goto out;
3704 }
27291e21
DH		 3705
3706 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3707 vcpu->guest_debug = dbg->control;
3708 /* enforce guest PER */
ef8f4f49 3709 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3710
3711 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3712 rc = kvm_s390_import_bp_data(vcpu, dbg);
3713 } else {
9daecfc6 3714 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3715 vcpu->arch.guestdbg.last_bp = 0;
3716 }
3717
3718 if (rc) {
3719 vcpu->guest_debug = 0;
3720 kvm_s390_clear_bp_data(vcpu);
9daecfc6 3721 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3722 }
3723
66b56562
CD		 3724out:
3725 vcpu_put(vcpu);
27291e21 3726 return rc;
b0c632db
HC		 3727}
3728
62d9f0db
MT		 3729int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3730 struct kvm_mp_state *mp_state)
3731{
fd232561
CD		 3732 int ret;
3733
3734 vcpu_load(vcpu);
3735
6352e4d2 3736 /* CHECK_STOP and LOAD are not supported yet */
fd232561
CD		 3737 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3738 KVM_MP_STATE_OPERATING;
3739
3740 vcpu_put(vcpu);
3741 return ret;
62d9f0db
MT		 3742}
3743
3744int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3745 struct kvm_mp_state *mp_state)
3746{
6352e4d2
DH		 3747 int rc = 0;
3748
e83dff5e
CD		 3749 vcpu_load(vcpu);
3750
6352e4d2
DH		 3751 /* user space knows about this interface - let it control the state */
3752 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3753
3754 switch (mp_state->mp_state) {
3755 case KVM_MP_STATE_STOPPED:
fe28c786 3756 rc = kvm_s390_vcpu_stop(vcpu);
6352e4d2
DH		 3757 break;
3758 case KVM_MP_STATE_OPERATING:
fe28c786 3759 rc = kvm_s390_vcpu_start(vcpu);
6352e4d2
DH		 3760 break;
3761 case KVM_MP_STATE_LOAD:
7c36a3fc
JF		 3762 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3763 rc = -ENXIO;
3764 break;
3765 }
3766 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
3767 break;
6352e4d2 3768 case KVM_MP_STATE_CHECK_STOP:
3b684a42 3769 fallthrough; /* CHECK_STOP and LOAD are not supported yet */
6352e4d2
DH		 3770 default:
3771 rc = -ENXIO;
3772 }
3773
e83dff5e 3774 vcpu_put(vcpu);
6352e4d2 3775 return rc;
62d9f0db
MT		 3776}
3777
8ad35755
DH		 3778static bool ibs_enabled(struct kvm_vcpu *vcpu)
3779{
8d5fb0dc 3780 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3781}
3782
2c70fe44
CB		 3783static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3784{
8ad35755 3785retry:
8e236546 3786 kvm_s390_vcpu_request_handled(vcpu);
2fa6e1e1 3787 if (!kvm_request_pending(vcpu))
586b7ccd 3788 return 0;
2c70fe44
CB		 3789 /*
3790 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 3791 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 3792 * This ensures that the ipte instruction for this request has
3793 * already finished. We might race against a second unmapper that
3794 * wants to set the blocking bit. Lets just retry the request loop.
3795 */
8ad35755 3796 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 3797 int rc;
b2d73b2a
MS		 3798 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3799 kvm_s390_get_prefix(vcpu),
3800 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 3801 if (rc) {
3802 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 3803 return rc;
aca411a4 3804 }
8ad35755 3805 goto retry;
2c70fe44 3806 }
8ad35755 3807
d3d692c8
DH		 3808 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3809 vcpu->arch.sie_block->ihcpu = 0xffff;
3810 goto retry;
3811 }
3812
8ad35755
DH		 3813 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3814 if (!ibs_enabled(vcpu)) {
3815 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
ef8f4f49 3816 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3817 }
3818 goto retry;
2c70fe44 3819 }
8ad35755
DH		 3820
3821 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3822 if (ibs_enabled(vcpu)) {
3823 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
9daecfc6 3824 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3825 }
3826 goto retry;
3827 }
3828
6502a34c
DH		 3829 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3830 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3831 goto retry;
3832 }
3833
190df4a2
CI		 3834 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3835 /*
c9f0a2b8 3836 * Disable CMM virtualization; we will emulate the ESSA
190df4a2
CI		 3837 * instruction manually, in order to provide additional
3838 * functionalities needed for live migration.
3839 */
3840 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3841 goto retry;
3842 }
3843
3844 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3845 /*
c9f0a2b8
JF		 3846 * Re-enable CMM virtualization if CMMA is available and
3847 * CMM has been used.
190df4a2
CI		 3848 */
3849 if ((vcpu->kvm->arch.use_cmma) &&
c9f0a2b8 3850 (vcpu->kvm->mm->context.uses_cmm))
190df4a2
CI		 3851 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3852 goto retry;
3853 }
3854
0759d068 3855 /* nothing to do, just clear the request */
72875d8a 3856 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3194cdb7
DH		 3857 /* we left the vsie handler, nothing to do, just clear the request */
3858 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
0759d068 3859
2c70fe44
CB		 3860 return 0;
3861}
3862
0e7def5f
DH		 3863void kvm_s390_set_tod_clock(struct kvm *kvm,
3864 const struct kvm_s390_vm_tod_clock *gtod)
8fa1696e
CW		 3865{
3866 struct kvm_vcpu *vcpu;
3867 struct kvm_s390_tod_clock_ext htod;
3868 int i;
3869
3870 mutex_lock(&kvm->lock);
3871 preempt_disable();
3872
3873 get_tod_clock_ext((char *)&htod);
3874
3875 kvm->arch.epoch = gtod->tod - htod.tod;
0e7def5f
DH		 3876 kvm->arch.epdx = 0;
3877 if (test_kvm_facility(kvm, 139)) {
3878 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3879 if (kvm->arch.epoch > gtod->tod)
3880 kvm->arch.epdx -= 1;
3881 }
8fa1696e
CW		 3882
3883 kvm_s390_vcpu_block_all(kvm);
3884 kvm_for_each_vcpu(i, vcpu, kvm) {
3885 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3886 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3887 }
3888
3889 kvm_s390_vcpu_unblock_all(kvm);
3890 preempt_enable();
3891 mutex_unlock(&kvm->lock);
3892}
3893
fa576c58
TH		 3894/**
3895 * kvm_arch_fault_in_page - fault-in guest page if necessary
3896 * @vcpu: The corresponding virtual cpu
3897 * @gpa: Guest physical address
3898 * @writable: Whether the page should be writable or not
3899 *
3900 * Make sure that a guest page has been faulted-in on the host.
3901 *
3902 * Return: Zero on success, negative error code otherwise.
3903 */
3904long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 3905{
527e30b4
MS		 3906 return gmap_fault(vcpu->arch.gmap, gpa,
3907 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 3908}
3909
3c038e6b
DD		 3910static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3911 unsigned long token)
3912{
3913 struct kvm_s390_interrupt inti;
383d0b05 3914 struct kvm_s390_irq irq;
3c038e6b
DD		 3915
3916 if (start_token) {
383d0b05
JF		 3917 irq.u.ext.ext_params2 = token;
3918 irq.type = KVM_S390_INT_PFAULT_INIT;
3919 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 3920 } else {
3921 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 3922 inti.parm64 = token;
3c038e6b
DD		 3923 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3924 }
3925}
3926
3927void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3928 struct kvm_async_pf *work)
3929{
3930 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3931 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3932}
3933
3934void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3935 struct kvm_async_pf *work)
3936{
3937 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3938 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3939}
3940
3941void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3942 struct kvm_async_pf *work)
3943{
3944 /* s390 will always inject the page directly */
3945}
3946
3947bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3948{
3949 /*
3950 * s390 will always inject the page directly,
3951 * but we still want check_async_completion to cleanup
3952 */
3953 return true;
3954}
3955
3956static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3957{
3958 hva_t hva;
3959 struct kvm_arch_async_pf arch;
3960 int rc;
3961
3962 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3963 return 0;
3964 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3965 vcpu->arch.pfault_compare)
3966 return 0;
3967 if (psw_extint_disabled(vcpu))
3968 return 0;
9a022067 3969 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b 3970 return 0;
b9224cd7 3971 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3c038e6b
DD		 3972 return 0;
3973 if (!vcpu->arch.gmap->pfault_enabled)
3974 return 0;
3975
81480cc1
HC		 3976 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3977 hva += current->thread.gmap_addr & ~PAGE_MASK;
3978 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 3979 return 0;
3980
3981 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3982 return rc;
3983}
3984
3fb4c40f 3985static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 3986{
3fb4c40f 3987 int rc, cpuflags;
e168bf8d 3988
3c038e6b
DD		 3989 /*
3990 * On s390 notifications for arriving pages will be delivered directly
3991 * to the guest but the house keeping for completed pfaults is
3992 * handled outside the worker.
3993 */
3994 kvm_check_async_pf_completion(vcpu);
3995
7ec7c8c7
CB		 3996 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3997 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 3998
3999 if (need_resched())
4000 schedule();
4001
d3a73acb 4002 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 4003 s390_handle_mcck();
4004
79395031
JF		 4005 if (!kvm_is_ucontrol(vcpu->kvm)) {
4006 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4007 if (rc)
4008 return rc;
4009 }
0ff31867 4010
2c70fe44
CB		 4011 rc = kvm_s390_handle_requests(vcpu);
4012 if (rc)
4013 return rc;
4014
27291e21
DH		 4015 if (guestdbg_enabled(vcpu)) {
4016 kvm_s390_backup_guest_per_regs(vcpu);
4017 kvm_s390_patch_guest_per_regs(vcpu);
4018 }
4019
9f30f621
MM		 4020 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask);
4021
b0c632db 4022 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 4023 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4024 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4025 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 4026
3fb4c40f
TH		 4027 return 0;
4028}
4029
492d8642
TH		 4030static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
4031{
56317920
DH		 4032 struct kvm_s390_pgm_info pgm_info = {
4033 .code = PGM_ADDRESSING,
4034 };
4035 u8 opcode, ilen;
492d8642
TH		 4036 int rc;
4037
4038 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4039 trace_kvm_s390_sie_fault(vcpu);
4040
4041 /*
4042 * We want to inject an addressing exception, which is defined as a
4043 * suppressing or terminating exception. However, since we came here
4044 * by a DAT access exception, the PSW still points to the faulting
4045 * instruction since DAT exceptions are nullifying. So we've got
4046 * to look up the current opcode to get the length of the instruction
4047 * to be able to forward the PSW.
4048 */
3fa8cad7 4049 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 4050 ilen = insn_length(opcode);
9b0d721a
DH		 4051 if (rc < 0) {
4052 return rc;
4053 } else if (rc) {
4054 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4055 * Forward by arbitrary ilc, injection will take care of
4056 * nullification if necessary.
4057 */
4058 pgm_info = vcpu->arch.pgm;
4059 ilen = 4;
4060 }
56317920
DH		 4061 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4062 kvm_s390_forward_psw(vcpu, ilen);
4063 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 4064}
4065
3fb4c40f
TH		 4066static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4067{
4d62fcc0
QH		 4068 struct mcck_volatile_info *mcck_info;
4069 struct sie_page *sie_page;
4070
2b29a9fd
DD		 4071 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4072 vcpu->arch.sie_block->icptcode);
4073 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4074
27291e21
DH		 4075 if (guestdbg_enabled(vcpu))
4076 kvm_s390_restore_guest_per_regs(vcpu);
4077
7ec7c8c7
CB		 4078 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4079 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf 4080
4d62fcc0
QH		 4081 if (exit_reason == -EINTR) {
4082 VCPU_EVENT(vcpu, 3, "%s", "machine check");
4083 sie_page = container_of(vcpu->arch.sie_block,
4084 struct sie_page, sie_block);
4085 mcck_info = &sie_page->mcck_info;
4086 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4087 return 0;
4088 }
4089
71f116bf
DH		 4090 if (vcpu->arch.sie_block->icptcode > 0) {
4091 int rc = kvm_handle_sie_intercept(vcpu);
4092
4093 if (rc != -EOPNOTSUPP)
4094 return rc;
4095 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4096 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4097 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4098 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4099 return -EREMOTE;
4100 } else if (exit_reason != -EFAULT) {
4101 vcpu->stat.exit_null++;
4102 return 0;
210b1607
TH		 4103 } else if (kvm_is_ucontrol(vcpu->kvm)) {
4104 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4105 vcpu->run->s390_ucontrol.trans_exc_code =
4106 current->thread.gmap_addr;
4107 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 4108 return -EREMOTE;
24eb3a82 4109 } else if (current->thread.gmap_pfault) {
3c038e6b 4110 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 4111 current->thread.gmap_pfault = 0;
71f116bf
DH		 4112 if (kvm_arch_setup_async_pf(vcpu))
4113 return 0;
4114 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 4115 }
71f116bf 4116 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 4117}
4118
3adae0b4 4119#define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
3fb4c40f
TH		 4120static int __vcpu_run(struct kvm_vcpu *vcpu)
4121{
4122 int rc, exit_reason;
c8aac234 4123 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
3fb4c40f 4124
800c1065
TH		 4125 /*
4126 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4127 * ning the guest), so that memslots (and other stuff) are protected
4128 */
4129 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4130
a76ccff6
TH		 4131 do {
4132 rc = vcpu_pre_run(vcpu);
4133 if (rc)
4134 break;
3fb4c40f 4135
800c1065 4136 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 4137 /*
4138 * As PF_VCPU will be used in fault handler, between
4139 * guest_enter and guest_exit should be no uaccess.
4140 */
0097d12e 4141 local_irq_disable();
6edaa530 4142 guest_enter_irqoff();
db0758b2 4143 __disable_cpu_timer_accounting(vcpu);
0097d12e 4144 local_irq_enable();
c8aac234
JF		 4145 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4146 memcpy(sie_page->pv_grregs,
4147 vcpu->run->s.regs.gprs,
4148 sizeof(sie_page->pv_grregs));
4149 }
a76ccff6
TH		 4150 exit_reason = sie64a(vcpu->arch.sie_block,
4151 vcpu->run->s.regs.gprs);
c8aac234
JF		 4152 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4153 memcpy(vcpu->run->s.regs.gprs,
4154 sie_page->pv_grregs,
4155 sizeof(sie_page->pv_grregs));
3adae0b4
JF		 4156 /*
4157 * We're not allowed to inject interrupts on intercepts
4158 * that leave the guest state in an "in-between" state
4159 * where the next SIE entry will do a continuation.
4160 * Fence interrupts in our "internal" PSW.
4161 */
4162 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4163 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4164 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4165 }
c8aac234 4166 }
0097d12e 4167 local_irq_disable();
db0758b2 4168 __enable_cpu_timer_accounting(vcpu);
6edaa530 4169 guest_exit_irqoff();
0097d12e 4170 local_irq_enable();
800c1065 4171 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 4172
4173 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 4174 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 4175
800c1065 4176 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 4177 return rc;
b0c632db
HC		 4178}
4179
811ea797 4180static void sync_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
b028ee3e 4181{
4d5f2c04 4182 struct runtime_instr_cb *riccb;
4e0b1ab7 4183 struct gs_cb *gscb;
4d5f2c04
CB		 4184
4185 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 4186 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 4187 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4188 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
b028ee3e 4189 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
b028ee3e
DH		 4190 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4191 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4192 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4193 }
4194 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4195 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4196 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4197 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 4198 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4199 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 4200 }
80cd8763
FZ		 4201 /*
4202 * If userspace sets the riccb (e.g. after migration) to a valid state,
4203 * we should enable RI here instead of doing the lazy enablement.
4204 */
4205 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04 4206 test_kvm_facility(vcpu->kvm, 64) &&
bb59c2da 4207 riccb->v &&
0c9d8683 4208 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 4209 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 4210 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 4211 }
4e0b1ab7
FZ		 4212 /*
4213 * If userspace sets the gscb (e.g. after migration) to non-zero,
4214 * we should enable GS here instead of doing the lazy enablement.
4215 */
4216 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4217 test_kvm_facility(vcpu->kvm, 133) &&
4218 gscb->gssm &&
4219 !vcpu->arch.gs_enabled) {
4220 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4221 vcpu->arch.sie_block->ecb |= ECB_GS;
4222 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4223 vcpu->arch.gs_enabled = 1;
80cd8763 4224 }
35b3fde6
CB		 4225 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4226 test_kvm_facility(vcpu->kvm, 82)) {
4227 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4228 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4229 }
811ea797
JF		 4230 if (MACHINE_HAS_GS) {
4231 preempt_disable();
4232 __ctl_set_bit(2, 4);
4233 if (current->thread.gs_cb) {
4234 vcpu->arch.host_gscb = current->thread.gs_cb;
4235 save_gs_cb(vcpu->arch.host_gscb);
4236 }
4237 if (vcpu->arch.gs_enabled) {
4238 current->thread.gs_cb = (struct gs_cb *)
4239 &vcpu->run->s.regs.gscb;
4240 restore_gs_cb(current->thread.gs_cb);
4241 }
4242 preempt_enable();
4243 }
4244 /* SIE will load etoken directly from SDNX and therefore kvm_run */
4245}
4246
4247static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4248{
4249 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4250 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4251 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4252 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4253 /* some control register changes require a tlb flush */
4254 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4255 }
4256 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4257 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4258 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4259 }
31d8b8d4
CB		 4260 save_access_regs(vcpu->arch.host_acrs);
4261 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 4262 /* save host (userspace) fprs/vrs */
4263 save_fpu_regs();
4264 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
4265 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
4266 if (MACHINE_HAS_VX)
4267 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
4268 else
4269 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
4270 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
4271 if (test_fp_ctl(current->thread.fpu.fpc))
4272 /* User space provided an invalid FPC, let's clear it */
4273 current->thread.fpu.fpc = 0;
811ea797
JF		 4274
4275 /* Sync fmt2 only data */
4276 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4277 sync_regs_fmt2(vcpu, kvm_run);
4278 } else {
4279 /*
4280 * In several places we have to modify our internal view to
4281 * not do things that are disallowed by the ultravisor. For
4282 * example we must not inject interrupts after specific exits
4283 * (e.g. 112 prefix page not secure). We do this by turning
4284 * off the machine check, external and I/O interrupt bits
4285 * of our PSW copy. To avoid getting validity intercepts, we
4286 * do only accept the condition code from userspace.
4287 */
4288 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4289 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4290 PSW_MASK_CC;
4291 }
4292
4293 kvm_run->kvm_dirty_regs = 0;
4294}
4295
4296static void store_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4297{
4298 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4299 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4300 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4301 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4e0b1ab7 4302 if (MACHINE_HAS_GS) {
4e0b1ab7 4303 __ctl_set_bit(2, 4);
811ea797
JF		 4304 if (vcpu->arch.gs_enabled)
4305 save_gs_cb(current->thread.gs_cb);
4306 preempt_disable();
4307 current->thread.gs_cb = vcpu->arch.host_gscb;
4308 restore_gs_cb(vcpu->arch.host_gscb);
4e0b1ab7 4309 preempt_enable();
811ea797
JF		 4310 if (!vcpu->arch.host_gscb)
4311 __ctl_clear_bit(2, 4);
4312 vcpu->arch.host_gscb = NULL;
4e0b1ab7 4313 }
811ea797 4314 /* SIE will save etoken directly into SDNX and therefore kvm_run */
b028ee3e
DH		 4315}
4316
4317static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4318{
4319 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4320 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4321 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4322 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 4323 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e 4324 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
b028ee3e
DH		 4325 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4326 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4327 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 4328 save_access_regs(vcpu->run->s.regs.acrs);
4329 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 4330 /* Save guest register state */
4331 save_fpu_regs();
4332 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4333 /* Restore will be done lazily at return */
4334 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4335 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
811ea797
JF		 4336 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
4337 store_regs_fmt2(vcpu, kvm_run);
b028ee3e
DH		 4338}
4339
b0c632db
HC		 4340int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4341{
8f2abe6a 4342 int rc;
b0c632db 4343
460df4c1
PB		 4344 if (kvm_run->immediate_exit)
4345 return -EINTR;
4346
200824f5
TH		 4347 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4348 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4349 return -EINVAL;
4350
accb757d
CD		 4351 vcpu_load(vcpu);
4352
27291e21
DH		 4353 if (guestdbg_exit_pending(vcpu)) {
4354 kvm_s390_prepare_debug_exit(vcpu);
accb757d
CD		 4355 rc = 0;
4356 goto out;
27291e21
DH		 4357 }
4358
20b7035c 4359 kvm_sigset_activate(vcpu);
b0c632db 4360
fe28c786
JF		 4361 /*
4362 * no need to check the return value of vcpu_start as it can only have
4363 * an error for protvirt, but protvirt means user cpu state
4364 */
6352e4d2
DH		 4365 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4366 kvm_s390_vcpu_start(vcpu);
4367 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 4368 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2 4369 vcpu->vcpu_id);
accb757d
CD		 4370 rc = -EINVAL;
4371 goto out;
6352e4d2 4372 }
b0c632db 4373
b028ee3e 4374 sync_regs(vcpu, kvm_run);
db0758b2 4375 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 4376
dab4079d 4377 might_fault();
a76ccff6 4378 rc = __vcpu_run(vcpu);
9ace903d 4379
b1d16c49
CE		 4380 if (signal_pending(current) && !rc) {
4381 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 4382 rc = -EINTR;
b1d16c49 4383 }
8f2abe6a 4384
27291e21
DH		 4385 if (guestdbg_exit_pending(vcpu) && !rc) {
4386 kvm_s390_prepare_debug_exit(vcpu);
4387 rc = 0;
4388 }
4389
8f2abe6a 4390 if (rc == -EREMOTE) {
71f116bf 4391 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 4392 rc = 0;
4393 }
b0c632db 4394
db0758b2 4395 disable_cpu_timer_accounting(vcpu);
b028ee3e 4396 store_regs(vcpu, kvm_run);
d7b0b5eb 4397
20b7035c 4398 kvm_sigset_deactivate(vcpu);
b0c632db 4399
b0c632db 4400 vcpu->stat.exit_userspace++;
accb757d
CD		 4401out:
4402 vcpu_put(vcpu);
7e8e6ab4 4403 return rc;
b0c632db
HC		 4404}
4405
b0c632db
HC		 4406/*
4407 * store status at address
4408 * we use have two special cases:
4409 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4410 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4411 */
d0bce605 4412int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 4413{
092670cd 4414 unsigned char archmode = 1;
9abc2a08 4415 freg_t fprs[NUM_FPRS];
fda902cb 4416 unsigned int px;
4287f247 4417 u64 clkcomp, cputm;
d0bce605 4418 int rc;
b0c632db 4419
d9a3a09a 4420 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 4421 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4422 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 4423 return -EFAULT;
d9a3a09a 4424 gpa = 0;
d0bce605
HC		 4425 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4426 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 4427 return -EFAULT;
d9a3a09a
MS		 4428 gpa = px;
4429 } else
4430 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 4431
4432 /* manually convert vector registers if necessary */
4433 if (MACHINE_HAS_VX) {
9522b37f 4434 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 4435 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4436 fprs, 128);
4437 } else {
4438 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 4439 vcpu->run->s.regs.fprs, 128);
9abc2a08 4440 }
d9a3a09a 4441 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 4442 vcpu->run->s.regs.gprs, 128);
d9a3a09a 4443 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 4444 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 4445 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 4446 &px, 4);
d9a3a09a 4447 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 4448 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 4449 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 4450 &vcpu->arch.sie_block->todpr, 4);
4287f247 4451 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 4452 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 4453 &cputm, 8);
178bd789 4454 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 4455 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 4456 &clkcomp, 8);
d9a3a09a 4457 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 4458 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 4459 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 4460 &vcpu->arch.sie_block->gcr, 128);
4461 return rc ? -EFAULT : 0;
b0c632db
HC		 4462}
4463
e879892c
TH		 4464int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4465{
4466 /*
4467 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 4468 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 4469 * it into the save area
4470 */
d0164ee2 4471 save_fpu_regs();
9abc2a08 4472 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 4473 save_access_regs(vcpu->run->s.regs.acrs);
4474
4475 return kvm_s390_store_status_unloaded(vcpu, addr);
4476}
4477
8ad35755
DH		 4478static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4479{
4480 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 4481 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 4482}
4483
4484static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4485{
4486 unsigned int i;
4487 struct kvm_vcpu *vcpu;
4488
4489 kvm_for_each_vcpu(i, vcpu, kvm) {
4490 __disable_ibs_on_vcpu(vcpu);
4491 }
4492}
4493
4494static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4495{
09a400e7
DH		 4496 if (!sclp.has_ibs)
4497 return;
8ad35755 4498 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 4499 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 4500}
4501
fe28c786 4502int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
6852d7b6 4503{
fe28c786 4504 int i, online_vcpus, r = 0, started_vcpus = 0;
8ad35755
DH		 4505
4506 if (!is_vcpu_stopped(vcpu))
fe28c786 4507 return 0;
8ad35755 4508
6852d7b6 4509 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 4510 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 4511 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 4512 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4513
fe28c786
JF		 4514 /* Let's tell the UV that we want to change into the operating state */
4515 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4516 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
4517 if (r) {
4518 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4519 return r;
4520 }
4521 }
4522
8ad35755
DH		 4523 for (i = 0; i < online_vcpus; i++) {
4524 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4525 started_vcpus++;
4526 }
4527
4528 if (started_vcpus == 0) {
4529 /* we're the only active VCPU -> speed it up */
4530 __enable_ibs_on_vcpu(vcpu);
4531 } else if (started_vcpus == 1) {
4532 /*
4533 * As we are starting a second VCPU, we have to disable
4534 * the IBS facility on all VCPUs to remove potentially
4535 * oustanding ENABLE requests.
4536 */
4537 __disable_ibs_on_all_vcpus(vcpu->kvm);
4538 }
4539
9daecfc6 4540 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
72f21820
CB		 4541 /*
4542 * The real PSW might have changed due to a RESTART interpreted by the
4543 * ultravisor. We block all interrupts and let the next sie exit
4544 * refresh our view.
4545 */
4546 if (kvm_s390_pv_cpu_is_protected(vcpu))
4547 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
8ad35755
DH		 4548 /*
4549 * Another VCPU might have used IBS while we were offline.
4550 * Let's play safe and flush the VCPU at startup.
4551 */
d3d692c8 4552 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 4553 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
fe28c786 4554 return 0;
6852d7b6
DH		 4555}
4556
fe28c786 4557int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
6852d7b6 4558{
fe28c786 4559 int i, online_vcpus, r = 0, started_vcpus = 0;
8ad35755
DH		 4560 struct kvm_vcpu *started_vcpu = NULL;
4561
4562 if (is_vcpu_stopped(vcpu))
fe28c786 4563 return 0;
8ad35755 4564
6852d7b6 4565 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 4566 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 4567 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 4568 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4569
fe28c786
JF		 4570 /* Let's tell the UV that we want to change into the stopped state */
4571 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4572 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
4573 if (r) {
4574 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4575 return r;
4576 }
4577 }
4578
32f5ff63 4579 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 4580 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 4581
ef8f4f49 4582 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
8ad35755
DH		 4583 __disable_ibs_on_vcpu(vcpu);
4584
4585 for (i = 0; i < online_vcpus; i++) {
4586 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4587 started_vcpus++;
4588 started_vcpu = vcpu->kvm->vcpus[i];
4589 }
4590 }
4591
4592 if (started_vcpus == 1) {
4593 /*
4594 * As we only have one VCPU left, we want to enable the
4595 * IBS facility for that VCPU to speed it up.
4596 */
4597 __enable_ibs_on_vcpu(started_vcpu);
4598 }
4599
433b9ee4 4600 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
fe28c786 4601 return 0;
6852d7b6
DH		 4602}
4603
d6712df9
CH		 4604static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4605 struct kvm_enable_cap *cap)
4606{
4607 int r;
4608
4609 if (cap->flags)
4610 return -EINVAL;
4611
4612 switch (cap->cap) {
fa6b7fe9
CH		 4613 case KVM_CAP_S390_CSS_SUPPORT:
4614 if (!vcpu->kvm->arch.css_support) {
4615 vcpu->kvm->arch.css_support = 1;
c92ea7b9 4616 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 4617 trace_kvm_s390_enable_css(vcpu->kvm);
4618 }
4619 r = 0;
4620 break;
d6712df9
CH		 4621 default:
4622 r = -EINVAL;
4623 break;
4624 }
4625 return r;
4626}
4627
19e12277
JF		 4628static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu,
4629 struct kvm_s390_mem_op *mop)
4630{
4631 void __user *uaddr = (void __user *)mop->buf;
4632 int r = 0;
4633
4634 if (mop->flags || !mop->size)
4635 return -EINVAL;
4636 if (mop->size + mop->sida_offset < mop->size)
4637 return -EINVAL;
4638 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
4639 return -E2BIG;
4640
4641 switch (mop->op) {
4642 case KVM_S390_MEMOP_SIDA_READ:
4643 if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
4644 mop->sida_offset), mop->size))
4645 r = -EFAULT;
4646
4647 break;
4648 case KVM_S390_MEMOP_SIDA_WRITE:
4649 if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
4650 mop->sida_offset), uaddr, mop->size))
4651 r = -EFAULT;
4652 break;
4653 }
4654 return r;
4655}
41408c28
TH		 4656static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4657 struct kvm_s390_mem_op *mop)
4658{
4659 void __user *uaddr = (void __user *)mop->buf;
4660 void *tmpbuf = NULL;
19e12277 4661 int r = 0;
41408c28
TH		 4662 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4663 | KVM_S390_MEMOP_F_CHECK_ONLY;
4664
a13b03bb 4665 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
41408c28
TH		 4666 return -EINVAL;
4667
4668 if (mop->size > MEM_OP_MAX_SIZE)
4669 return -E2BIG;
4670
19e12277
JF		 4671 if (kvm_s390_pv_cpu_is_protected(vcpu))
4672 return -EINVAL;
4673
41408c28
TH		 4674 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4675 tmpbuf = vmalloc(mop->size);
4676 if (!tmpbuf)
4677 return -ENOMEM;
4678 }
4679
41408c28
TH		 4680 switch (mop->op) {
4681 case KVM_S390_MEMOP_LOGICAL_READ:
4682 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 4683 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4684 mop->size, GACC_FETCH);
41408c28
TH		 4685 break;
4686 }
4687 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4688 if (r == 0) {
4689 if (copy_to_user(uaddr, tmpbuf, mop->size))
4690 r = -EFAULT;
4691 }
4692 break;
4693 case KVM_S390_MEMOP_LOGICAL_WRITE:
4694 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 4695 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4696 mop->size, GACC_STORE);
41408c28
TH		 4697 break;
4698 }
4699 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4700 r = -EFAULT;
4701 break;
4702 }
4703 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4704 break;
41408c28
TH		 4705 }
4706
41408c28
TH		 4707 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4708 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4709
4710 vfree(tmpbuf);
4711 return r;
4712}
4713
19e12277
JF		 4714static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu,
4715 struct kvm_s390_mem_op *mop)
4716{
4717 int r, srcu_idx;
4718
4719 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4720
4721 switch (mop->op) {
4722 case KVM_S390_MEMOP_LOGICAL_READ:
4723 case KVM_S390_MEMOP_LOGICAL_WRITE:
4724 r = kvm_s390_guest_mem_op(vcpu, mop);
4725 break;
4726 case KVM_S390_MEMOP_SIDA_READ:
4727 case KVM_S390_MEMOP_SIDA_WRITE:
4728 /* we are locked against sida going away by the vcpu->mutex */
4729 r = kvm_s390_guest_sida_op(vcpu, mop);
4730 break;
4731 default:
4732 r = -EINVAL;
4733 }
4734
4735 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4736 return r;
4737}
4738
5cb0944c
PB		 4739long kvm_arch_vcpu_async_ioctl(struct file *filp,
4740 unsigned int ioctl, unsigned long arg)
b0c632db
HC		 4741{
4742 struct kvm_vcpu *vcpu = filp->private_data;
4743 void __user *argp = (void __user *)arg;
4744
93736624 4745 switch (ioctl) {
47b43c52
JF		 4746 case KVM_S390_IRQ: {
4747 struct kvm_s390_irq s390irq;
4748
47b43c52 4749 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
9b062471
CD		 4750 return -EFAULT;
4751 return kvm_s390_inject_vcpu(vcpu, &s390irq);
47b43c52 4752 }
93736624 4753 case KVM_S390_INTERRUPT: {
ba5c1e9b 4754 struct kvm_s390_interrupt s390int;
53936b5b 4755 struct kvm_s390_irq s390irq = {};
ba5c1e9b
CO		 4756
4757 if (copy_from_user(&s390int, argp, sizeof(s390int)))
9b062471 4758 return -EFAULT;
383d0b05
JF		 4759 if (s390int_to_s390irq(&s390int, &s390irq))
4760 return -EINVAL;
9b062471 4761 return kvm_s390_inject_vcpu(vcpu, &s390irq);
ba5c1e9b 4762 }
9b062471 4763 }
5cb0944c
PB		 4764 return -ENOIOCTLCMD;
4765}
4766
4767long kvm_arch_vcpu_ioctl(struct file *filp,
4768 unsigned int ioctl, unsigned long arg)
4769{
4770 struct kvm_vcpu *vcpu = filp->private_data;
4771 void __user *argp = (void __user *)arg;
4772 int idx;
4773 long r;
8a8378fa 4774 u16 rc, rrc;
9b062471
CD		 4775
4776 vcpu_load(vcpu);
4777
4778 switch (ioctl) {
b0c632db 4779 case KVM_S390_STORE_STATUS:
800c1065 4780 idx = srcu_read_lock(&vcpu->kvm->srcu);
55680890 4781 r = kvm_s390_store_status_unloaded(vcpu, arg);
800c1065 4782 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 4783 break;
b0c632db
HC		 4784 case KVM_S390_SET_INITIAL_PSW: {
4785 psw_t psw;
4786
bc923cc9 4787 r = -EFAULT;
b0c632db 4788 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 4789 break;
4790 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4791 break;
b0c632db 4792 }
7de3f142
JF		 4793 case KVM_S390_CLEAR_RESET:
4794 r = 0;
4795 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
8a8378fa
JF		 4796 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4797 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4798 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
4799 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
4800 rc, rrc);
4801 }
7de3f142 4802 break;
b0c632db 4803 case KVM_S390_INITIAL_RESET:
7de3f142
JF		 4804 r = 0;
4805 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
8a8378fa
JF		 4806 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4807 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4808 UVC_CMD_CPU_RESET_INITIAL,
4809 &rc, &rrc);
4810 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
4811 rc, rrc);
4812 }
7de3f142
JF		 4813 break;
4814 case KVM_S390_NORMAL_RESET:
4815 r = 0;
4816 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
8a8378fa
JF		 4817 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4818 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4819 UVC_CMD_CPU_RESET, &rc, &rrc);
4820 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
4821 rc, rrc);
4822 }
bc923cc9 4823 break;
14eebd91
CO		 4824 case KVM_SET_ONE_REG:
4825 case KVM_GET_ONE_REG: {
4826 struct kvm_one_reg reg;
68cf7b1f
JF		 4827 r = -EINVAL;
4828 if (kvm_s390_pv_cpu_is_protected(vcpu))
4829 break;
14eebd91
CO		 4830 r = -EFAULT;
4831 if (copy_from_user(&reg, argp, sizeof(reg)))
4832 break;
4833 if (ioctl == KVM_SET_ONE_REG)
4834 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4835 else
4836 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4837 break;
4838 }
27e0393f
CO		 4839#ifdef CONFIG_KVM_S390_UCONTROL
4840 case KVM_S390_UCAS_MAP: {
4841 struct kvm_s390_ucas_mapping ucasmap;
4842
4843 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4844 r = -EFAULT;
4845 break;
4846 }
4847
4848 if (!kvm_is_ucontrol(vcpu->kvm)) {
4849 r = -EINVAL;
4850 break;
4851 }
4852
4853 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4854 ucasmap.vcpu_addr, ucasmap.length);
4855 break;
4856 }
4857 case KVM_S390_UCAS_UNMAP: {
4858 struct kvm_s390_ucas_mapping ucasmap;
4859
4860 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4861 r = -EFAULT;
4862 break;
4863 }
4864
4865 if (!kvm_is_ucontrol(vcpu->kvm)) {
4866 r = -EINVAL;
4867 break;
4868 }
4869
4870 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4871 ucasmap.length);
4872 break;
4873 }
4874#endif
ccc7910f 4875 case KVM_S390_VCPU_FAULT: {
527e30b4 4876 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 4877 break;
4878 }
d6712df9
CH		 4879 case KVM_ENABLE_CAP:
4880 {
4881 struct kvm_enable_cap cap;
4882 r = -EFAULT;
4883 if (copy_from_user(&cap, argp, sizeof(cap)))
4884 break;
4885 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4886 break;
4887 }
41408c28
TH		 4888 case KVM_S390_MEM_OP: {
4889 struct kvm_s390_mem_op mem_op;
4890
4891 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
19e12277 4892 r = kvm_s390_guest_memsida_op(vcpu, &mem_op);
41408c28
TH		 4893 else
4894 r = -EFAULT;
4895 break;
4896 }
816c7667
JF		 4897 case KVM_S390_SET_IRQ_STATE: {
4898 struct kvm_s390_irq_state irq_state;
4899
4900 r = -EFAULT;
4901 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4902 break;
4903 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4904 irq_state.len == 0 ||
4905 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4906 r = -EINVAL;
4907 break;
4908 }
bb64da9a 4909 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 4910 r = kvm_s390_set_irq_state(vcpu,
4911 (void __user *) irq_state.buf,
4912 irq_state.len);
4913 break;
4914 }
4915 case KVM_S390_GET_IRQ_STATE: {
4916 struct kvm_s390_irq_state irq_state;
4917
4918 r = -EFAULT;
4919 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4920 break;
4921 if (irq_state.len == 0) {
4922 r = -EINVAL;
4923 break;
4924 }
bb64da9a 4925 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 4926 r = kvm_s390_get_irq_state(vcpu,
4927 (__u8 __user *) irq_state.buf,
4928 irq_state.len);
4929 break;
4930 }
b0c632db 4931 default:
3e6afcf1 4932 r = -ENOTTY;
b0c632db 4933 }
9b062471
CD		 4934
4935 vcpu_put(vcpu);
bc923cc9 4936 return r;
b0c632db
HC		 4937}
4938
1499fa80 4939vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5b1c1493
CO		 4940{
4941#ifdef CONFIG_KVM_S390_UCONTROL
4942 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4943 && (kvm_is_ucontrol(vcpu->kvm))) {
4944 vmf->page = virt_to_page(vcpu->arch.sie_block);
4945 get_page(vmf->page);
4946 return 0;
4947 }
4948#endif
4949 return VM_FAULT_SIGBUS;
4950}
4951
b0c632db 4952/* Section: memory related */
f7784b8e
MT		 4953int kvm_arch_prepare_memory_region(struct kvm *kvm,
4954 struct kvm_memory_slot *memslot,
09170a49 4955 const struct kvm_userspace_memory_region *mem,
7b6195a9 4956 enum kvm_mr_change change)
b0c632db 4957{
dd2887e7
NW		 4958 /* A few sanity checks. We can have memory slots which have to be
4959 located/ended at a segment boundary (1MB). The memory in userland is
4960 ok to be fragmented into various different vmas. It is okay to mmap()
4961 and munmap() stuff in this slot after doing this call at any time */
b0c632db 4962
598841ca 4963 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 4964 return -EINVAL;
4965
598841ca 4966 if (mem->memory_size & 0xffffful)
b0c632db
HC		 4967 return -EINVAL;
4968
a3a92c31
DD		 4969 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4970 return -EINVAL;
4971
29b40f10
JF		 4972 /* When we are protected, we should not change the memory slots */
4973 if (kvm_s390_pv_get_handle(kvm))
4974 return -EINVAL;
f7784b8e
MT		 4975 return 0;
4976}
4977
4978void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 4979 const struct kvm_userspace_memory_region *mem,
9d4c197c 4980 struct kvm_memory_slot *old,
f36f3f28 4981 const struct kvm_memory_slot *new,
8482644a 4982 enum kvm_mr_change change)
f7784b8e 4983{
19ec166c 4984 int rc = 0;
598841ca 4985
19ec166c
CB		 4986 switch (change) {
4987 case KVM_MR_DELETE:
4988 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4989 old->npages * PAGE_SIZE);
4990 break;
4991 case KVM_MR_MOVE:
4992 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4993 old->npages * PAGE_SIZE);
4994 if (rc)
4995 break;
3b684a42 4996 fallthrough;
19ec166c
CB		 4997 case KVM_MR_CREATE:
4998 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4999 mem->guest_phys_addr, mem->memory_size);
5000 break;
5001 case KVM_MR_FLAGS_ONLY:
5002 break;
5003 default:
5004 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
5005 }
598841ca 5006 if (rc)
ea2cdd27 5007 pr_warn("failed to commit memory region\n");
598841ca 5008 return;
b0c632db
HC		 5009}
5010
60a37709
AY		 5011static inline unsigned long nonhyp_mask(int i)
5012{
5013 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5014
5015 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5016}
5017
3491caf2
CB		 5018void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
5019{
5020 vcpu->valid_wakeup = false;
5021}
5022
b0c632db
HC		 5023static int __init kvm_s390_init(void)
5024{
60a37709
AY		 5025 int i;
5026
07197fd0 5027 if (!sclp.has_sief2) {
8d43d570 5028 pr_info("SIE is not available\n");
07197fd0
DH		 5029 return -ENODEV;
5030 }
5031
a4499382 5032 if (nested && hpage) {
8d43d570 5033 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
a4499382
JF		 5034 return -EINVAL;
5035 }
5036
60a37709 5037 for (i = 0; i < 16; i++)
c3b9e3e1 5038 kvm_s390_fac_base[i] |=
60a37709
AY		 5039 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
5040
9d8d5786 5041 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 5042}
5043
5044static void __exit kvm_s390_exit(void)
5045{
5046 kvm_exit();
5047}
5048
5049module_init(kvm_s390_init);
5050module_exit(kvm_s390_exit);
566af940
CH		 5051
5052/*
5053 * Enable autoloading of the kvm module.
5054 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5055 * since x86 takes a different approach.
5056 */
5057#include <linux/miscdevice.h>
5058MODULE_ALIAS_MISCDEV(KVM_MINOR);
5059MODULE_ALIAS("devname:kvm");
A mirror of Linus' kernel repository