1 // SPDX-License-Identifier: GPL-2.0
3 * AMD Encrypted Register State Support
5 * Author: Joerg Roedel <jroedel@suse.de>
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
15 #include <asm/pgtable_types.h>
17 #include <asm/trapnr.h>
18 #include <asm/trap_pf.h>
19 #include <asm/msr-index.h>
20 #include <asm/fpu/xcr.h>
21 #include <asm/ptrace.h>
23 #include <asm/cpuid.h>
28 struct ghcb boot_ghcb_page
__aligned(PAGE_SIZE
);
29 struct ghcb
*boot_ghcb
;
32 * Copy a version of this function here - insn-eval.c can't be used in
33 * pre-decompression code.
35 static bool insn_has_rep_prefix(struct insn
*insn
)
40 insn_get_prefixes(insn
);
42 for_each_insn_prefix(insn
, i
, p
) {
43 if (p
== 0xf2 || p
== 0xf3)
51 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
52 * doesn't use segments.
54 static unsigned long insn_get_seg_base(struct pt_regs
*regs
, int seg_reg_idx
)
59 static inline u64
sev_es_rd_ghcb_msr(void)
63 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB
, &m
);
68 static inline void sev_es_wr_ghcb_msr(u64 val
)
73 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB
, &m
);
76 static enum es_result
vc_decode_insn(struct es_em_ctxt
*ctxt
)
78 char buffer
[MAX_INSN_SIZE
];
81 memcpy(buffer
, (unsigned char *)ctxt
->regs
->ip
, MAX_INSN_SIZE
);
83 ret
= insn_decode(&ctxt
->insn
, buffer
, MAX_INSN_SIZE
, INSN_MODE_64
);
85 return ES_DECODE_FAILED
;
90 static enum es_result
vc_write_mem(struct es_em_ctxt
*ctxt
,
91 void *dst
, char *buf
, size_t size
)
93 memcpy(dst
, buf
, size
);
98 static enum es_result
vc_read_mem(struct es_em_ctxt
*ctxt
,
99 void *src
, char *buf
, size_t size
)
101 memcpy(buf
, src
, size
);
109 #define __BOOT_COMPRESSED
111 /* Basic instruction decoding support needed */
112 #include "../../lib/inat.c"
113 #include "../../lib/insn.c"
115 /* Include code for early handlers */
116 #include "../../kernel/sev-shared.c"
118 bool sev_snp_enabled(void)
120 return sev_status
& MSR_AMD64_SEV_SNP_ENABLED
;
123 static void __page_state_change(unsigned long paddr
, enum psc_op op
)
127 if (!sev_snp_enabled())
131 * If private -> shared then invalidate the page before requesting the
132 * state change in the RMP table.
134 if (op
== SNP_PAGE_STATE_SHARED
&& pvalidate(paddr
, RMP_PG_SIZE_4K
, 0))
135 sev_es_terminate(SEV_TERM_SET_LINUX
, GHCB_TERM_PVALIDATE
);
137 /* Issue VMGEXIT to change the page state in RMP table. */
138 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr
>> PAGE_SHIFT
, op
));
141 /* Read the response of the VMGEXIT. */
142 val
= sev_es_rd_ghcb_msr();
143 if ((GHCB_RESP_CODE(val
) != GHCB_MSR_PSC_RESP
) || GHCB_MSR_PSC_RESP_VAL(val
))
144 sev_es_terminate(SEV_TERM_SET_LINUX
, GHCB_TERM_PSC
);
147 * Now that page state is changed in the RMP table, validate it so that it is
148 * consistent with the RMP entry.
150 if (op
== SNP_PAGE_STATE_PRIVATE
&& pvalidate(paddr
, RMP_PG_SIZE_4K
, 1))
151 sev_es_terminate(SEV_TERM_SET_LINUX
, GHCB_TERM_PVALIDATE
);
154 void snp_set_page_private(unsigned long paddr
)
156 __page_state_change(paddr
, SNP_PAGE_STATE_PRIVATE
);
159 void snp_set_page_shared(unsigned long paddr
)
161 __page_state_change(paddr
, SNP_PAGE_STATE_SHARED
);
164 static bool early_setup_ghcb(void)
166 if (set_page_decrypted((unsigned long)&boot_ghcb_page
))
169 /* Page is now mapped decrypted, clear it */
170 memset(&boot_ghcb_page
, 0, sizeof(boot_ghcb_page
));
172 boot_ghcb
= &boot_ghcb_page
;
174 /* Initialize lookup tables for the instruction decoder */
177 /* SNP guest requires the GHCB GPA must be registered */
178 if (sev_snp_enabled())
179 snp_register_ghcb_early(__pa(&boot_ghcb_page
));
184 static phys_addr_t
__snp_accept_memory(struct snp_psc_desc
*desc
,
185 phys_addr_t pa
, phys_addr_t pa_end
)
192 memset(hdr
, 0, sizeof(*hdr
));
197 while (pa
< pa_end
&& i
< VMGEXIT_PSC_MAX_ENTRY
) {
200 e
->gfn
= pa
>> PAGE_SHIFT
;
201 e
->operation
= SNP_PAGE_STATE_PRIVATE
;
202 if (IS_ALIGNED(pa
, PMD_SIZE
) && (pa_end
- pa
) >= PMD_SIZE
) {
203 e
->pagesize
= RMP_PG_SIZE_2M
;
206 e
->pagesize
= RMP_PG_SIZE_4K
;
214 if (vmgexit_psc(boot_ghcb
, desc
))
215 sev_es_terminate(SEV_TERM_SET_LINUX
, GHCB_TERM_PSC
);
217 pvalidate_pages(desc
);
222 void snp_accept_memory(phys_addr_t start
, phys_addr_t end
)
224 struct snp_psc_desc desc
= {};
228 if (!boot_ghcb
&& !early_setup_ghcb())
229 sev_es_terminate(SEV_TERM_SET_LINUX
, GHCB_TERM_PSC
);
233 pa
= __snp_accept_memory(&desc
, pa
, end
);
236 void sev_es_shutdown_ghcb(void)
241 if (!sev_es_check_cpu_features())
242 error("SEV-ES CPU Features missing.");
245 * GHCB Page must be flushed from the cache and mapped encrypted again.
246 * Otherwise the running kernel will see strange cache effects when
247 * trying to use that page.
249 if (set_page_encrypted((unsigned long)&boot_ghcb_page
))
250 error("Can't map GHCB page encrypted");
253 * GHCB page is mapped encrypted again and flushed from the cache.
254 * Mark it non-present now to catch bugs when #VC exceptions trigger
257 if (set_page_non_present((unsigned long)&boot_ghcb_page
))
258 error("Can't unmap GHCB page");
261 static void __noreturn
sev_es_ghcb_terminate(struct ghcb
*ghcb
, unsigned int set
,
262 unsigned int reason
, u64 exit_info_2
)
264 u64 exit_info_1
= SVM_VMGEXIT_TERM_REASON(set
, reason
);
266 vc_ghcb_invalidate(ghcb
);
267 ghcb_set_sw_exit_code(ghcb
, SVM_VMGEXIT_TERM_REQUEST
);
268 ghcb_set_sw_exit_info_1(ghcb
, exit_info_1
);
269 ghcb_set_sw_exit_info_2(ghcb
, exit_info_2
);
271 sev_es_wr_ghcb_msr(__pa(ghcb
));
275 asm volatile("hlt\n" : : : "memory");
278 bool sev_es_check_ghcb_fault(unsigned long address
)
280 /* Check whether the fault was on the GHCB page */
281 return ((address
& PAGE_MASK
) == (unsigned long)&boot_ghcb_page
);
284 void do_boot_stage2_vc(struct pt_regs
*regs
, unsigned long exit_code
)
286 struct es_em_ctxt ctxt
;
287 enum es_result result
;
289 if (!boot_ghcb
&& !early_setup_ghcb())
290 sev_es_terminate(SEV_TERM_SET_GEN
, GHCB_SEV_ES_GEN_REQ
);
292 vc_ghcb_invalidate(boot_ghcb
);
293 result
= vc_init_em_ctxt(&ctxt
, regs
, exit_code
);
299 case SVM_EXIT_RDTSCP
:
300 result
= vc_handle_rdtsc(boot_ghcb
, &ctxt
, exit_code
);
303 result
= vc_handle_ioio(boot_ghcb
, &ctxt
);
306 result
= vc_handle_cpuid(boot_ghcb
, &ctxt
);
309 result
= ES_UNSUPPORTED
;
315 vc_finish_insn(&ctxt
);
316 else if (result
!= ES_RETRY
)
317 sev_es_terminate(SEV_TERM_SET_GEN
, GHCB_SEV_ES_GEN_REQ
);
320 static void enforce_vmpl0(void)
326 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
327 * higher) privilege level. Here, clear the VMPL1 permission mask of the
328 * GHCB page. If the guest is not running at VMPL0, this will fail.
330 * If the guest is running at VMPL0, it will succeed. Even if that operation
331 * modifies permission bits, it is still ok to do so currently because Linux
332 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
333 * changing is a don't-care.
336 if (rmpadjust((unsigned long)&boot_ghcb_page
, RMP_PG_SIZE_4K
, attrs
))
337 sev_es_terminate(SEV_TERM_SET_LINUX
, GHCB_TERM_NOT_VMPL0
);
341 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
342 * guest side implementation for proper functioning of the guest. If any
343 * of these features are enabled in the hypervisor but are lacking guest
344 * side implementation, the behavior of the guest will be undefined. The
345 * guest could fail in non-obvious way making it difficult to debug.
347 * As the behavior of reserved feature bits is unknown to be on the
348 * safe side add them to the required features mask.
350 #define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
351 MSR_AMD64_SNP_REFLECT_VC | \
352 MSR_AMD64_SNP_RESTRICTED_INJ | \
353 MSR_AMD64_SNP_ALT_INJ | \
354 MSR_AMD64_SNP_DEBUG_SWAP | \
355 MSR_AMD64_SNP_VMPL_SSS | \
356 MSR_AMD64_SNP_SECURE_TSC | \
357 MSR_AMD64_SNP_VMGEXIT_PARAM | \
358 MSR_AMD64_SNP_VMSA_REG_PROTECTION | \
359 MSR_AMD64_SNP_RESERVED_BIT13 | \
360 MSR_AMD64_SNP_RESERVED_BIT15 | \
361 MSR_AMD64_SNP_RESERVED_MASK)
364 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
365 * by the guest kernel. As and when a new feature is implemented in the
366 * guest kernel, a corresponding bit should be added to the mask.
368 #define SNP_FEATURES_PRESENT MSR_AMD64_SNP_DEBUG_SWAP
370 u64
snp_get_unsupported_features(u64 status
)
372 if (!(status
& MSR_AMD64_SEV_SNP_ENABLED
))
375 return status
& SNP_FEATURES_IMPL_REQ
& ~SNP_FEATURES_PRESENT
;
378 void snp_check_features(void)
383 * Terminate the boot if hypervisor has enabled any feature lacking
384 * guest side implementation. Pass on the unsupported features mask through
385 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
386 * as part of the guest boot failure.
388 unsupported
= snp_get_unsupported_features(sev_status
);
390 if (ghcb_version
< 2 || (!boot_ghcb
&& !early_setup_ghcb()))
391 sev_es_terminate(SEV_TERM_SET_GEN
, GHCB_SNP_UNSUPPORTED
);
393 sev_es_ghcb_terminate(boot_ghcb
, SEV_TERM_SET_GEN
,
394 GHCB_SNP_UNSUPPORTED
, unsupported
);
399 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
401 * Returns < 0 if SEV is not supported, otherwise the position of the
402 * encryption bit in the page table descriptors.
404 static int sev_check_cpu_support(void)
406 unsigned int eax
, ebx
, ecx
, edx
;
408 /* Check for the SME/SEV support leaf */
411 native_cpuid(&eax
, &ebx
, &ecx
, &edx
);
412 if (eax
< 0x8000001f)
416 * Check for the SME/SEV feature:
417 * CPUID Fn8000_001F[EAX]
418 * - Bit 0 - Secure Memory Encryption support
419 * - Bit 1 - Secure Encrypted Virtualization support
420 * CPUID Fn8000_001F[EBX]
421 * - Bits 5:0 - Pagetable bit position used to indicate encryption
425 native_cpuid(&eax
, &ebx
, &ecx
, &edx
);
426 /* Check whether SEV is supported */
433 void sev_enable(struct boot_params
*bp
)
440 * bp->cc_blob_address should only be set by boot/compressed kernel.
441 * Initialize it to 0 to ensure that uninitialized values from
442 * buggy bootloaders aren't propagated.
445 bp
->cc_blob_address
= 0;
448 * Do an initial SEV capability check before snp_init() which
449 * loads the CPUID page and the same checks afterwards are done
450 * without the hypervisor and are trustworthy.
452 * If the HV fakes SEV support, the guest will crash'n'burn
453 * which is good enough.
456 if (sev_check_cpu_support() < 0)
460 * Setup/preliminary detection of SNP. This will be sanity-checked
461 * against CPUID/MSR values later.
465 /* Now repeat the checks with the SNP CPUID table. */
467 bitpos
= sev_check_cpu_support();
470 error("SEV-SNP support indicated by CC blob, but not CPUID.");
474 /* Set the SME mask if this is an SEV guest. */
475 boot_rdmsr(MSR_AMD64_SEV
, &m
);
477 if (!(sev_status
& MSR_AMD64_SEV_ENABLED
))
480 /* Negotiate the GHCB protocol version. */
481 if (sev_status
& MSR_AMD64_SEV_ES_ENABLED
) {
482 if (!sev_es_negotiate_protocol())
483 sev_es_terminate(SEV_TERM_SET_GEN
, GHCB_SEV_ES_PROT_UNSUPPORTED
);
487 * SNP is supported in v2 of the GHCB spec which mandates support for HV
490 if (sev_status
& MSR_AMD64_SEV_SNP_ENABLED
) {
491 if (!(get_hv_features() & GHCB_HV_FT_SNP
))
492 sev_es_terminate(SEV_TERM_SET_GEN
, GHCB_SNP_UNSUPPORTED
);
497 if (snp
&& !(sev_status
& MSR_AMD64_SEV_SNP_ENABLED
))
498 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
500 sme_me_mask
= BIT_ULL(bitpos
);
504 * sev_get_status - Retrieve the SEV status mask
506 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
509 u64
sev_get_status(void)
513 if (sev_check_cpu_support() < 0)
516 boot_rdmsr(MSR_AMD64_SEV
, &m
);
520 /* Search for Confidential Computing blob in the EFI config table. */
521 static struct cc_blob_sev_info
*find_cc_blob_efi(struct boot_params
*bp
)
523 unsigned long cfg_table_pa
;
524 unsigned int cfg_table_len
;
527 ret
= efi_get_conf_table(bp
, &cfg_table_pa
, &cfg_table_len
);
531 return (struct cc_blob_sev_info
*)efi_find_vendor_table(bp
, cfg_table_pa
,
537 * Initial set up of SNP relies on information provided by the
538 * Confidential Computing blob, which can be passed to the boot kernel
539 * by firmware/bootloader in the following ways:
541 * - via an entry in the EFI config table
542 * - via a setup_data structure, as defined by the Linux Boot Protocol
544 * Scan for the blob in that order.
546 static struct cc_blob_sev_info
*find_cc_blob(struct boot_params
*bp
)
548 struct cc_blob_sev_info
*cc_info
;
550 cc_info
= find_cc_blob_efi(bp
);
554 cc_info
= find_cc_blob_setup_data(bp
);
559 if (cc_info
->magic
!= CC_BLOB_SEV_HDR_MAGIC
)
560 sev_es_terminate(SEV_TERM_SET_GEN
, GHCB_SNP_UNSUPPORTED
);
566 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
567 * will verify the SNP CPUID/MSR bits.
569 bool snp_init(struct boot_params
*bp
)
571 struct cc_blob_sev_info
*cc_info
;
576 cc_info
= find_cc_blob(bp
);
581 * If a SNP-specific Confidential Computing blob is present, then
582 * firmware/bootloader have indicated SNP support. Verifying this
583 * involves CPUID checks which will be more reliable if the SNP
584 * CPUID table is used. See comments over snp_setup_cpuid_table() for
587 setup_cpuid_table(cc_info
);
590 * Pass run-time kernel a pointer to CC info via boot_params so EFI
591 * config table doesn't need to be searched again during early startup
594 bp
->cc_blob_address
= (u32
)(unsigned long)cc_info
;
599 void sev_prep_identity_maps(unsigned long top_level_pgt
)
602 * The Confidential Computing blob is used very early in uncompressed
603 * kernel to find the in-memory CPUID table to handle CPUID
604 * instructions. Make sure an identity-mapping exists so it can be
605 * accessed after switchover.
607 if (sev_snp_enabled()) {
608 unsigned long cc_info_pa
= boot_params
->cc_blob_address
;
609 struct cc_blob_sev_info
*cc_info
;
611 kernel_add_identity_map(cc_info_pa
, cc_info_pa
+ sizeof(*cc_info
));
613 cc_info
= (struct cc_blob_sev_info
*)cc_info_pa
;
614 kernel_add_identity_map(cc_info
->cpuid_phys
, cc_info
->cpuid_phys
+ cc_info
->cpuid_len
);
617 sev_verify_cbit(top_level_pgt
);