Merge tag 'kvm-x86-lam-6.8' of https://github.com/kvm-x86/linux into HEAD

KVM x86 support for virtualizing Linear Address Masking (LAM)

Add KVM support for Linear Address Masking (LAM).  LAM tweaks the canonicality
checks for most virtual address usage in 64-bit mode, such that only the most
significant bit of the untranslated address bits must match the polarity of the
last translated address bit.  This allows software to use ignored, untranslated
address bits for metadata, e.g. to efficiently tag pointers for address
sanitization.

LAM can be enabled separately for user pointers and supervisor pointers, and
for userspace LAM can be select between 48-bit and 57-bit masking

 - 48-bit LAM: metadata bits 62:48, i.e. LAM width of 15.
 - 57-bit LAM: metadata bits 62:57, i.e. LAM width of 6.

For user pointers, LAM enabling utilizes two previously-reserved high bits from
CR3 (similar to how PCID_NOFLUSH uses bit 63): LAM_U48 and LAM_U57, bits 62 and
61 respectively.  Note, if LAM_57 is set, LAM_U48 is ignored, i.e.:

 - CR3.LAM_U48=0 && CR3.LAM_U57=0 == LAM disabled for user pointers
 - CR3.LAM_U48=1 && CR3.LAM_U57=0 == LAM-48 enabled for user pointers
 - CR3.LAM_U48=x && CR3.LAM_U57=1 == LAM-57 enabled for user pointers

For supervisor pointers, LAM is controlled by a single bit, CR4.LAM_SUP, with
the 48-bit versus 57-bit LAM behavior following the current paging mode, i.e.:

 - CR4.LAM_SUP=0 && CR4.LA57=x == LAM disabled for supervisor pointers
 - CR4.LAM_SUP=1 && CR4.LA57=0 == LAM-48 enabled for supervisor pointers
 - CR4.LAM_SUP=1 && CR4.LA57=1 == LAM-57 enabled for supervisor pointers

The modified LAM canonicality checks:
 - LAM_S48                : [ 1 ][ metadata ][ 1 ]
                              63               47
 - LAM_U48                : [ 0 ][ metadata ][ 0 ]
                              63               47
 - LAM_S57                : [ 1 ][ metadata ][ 1 ]
                              63               56
 - LAM_U57 + 5-lvl paging : [ 0 ][ metadata ][ 0 ]
                              63               56
 - LAM_U57 + 4-lvl paging : [ 0 ][ metadata ][ 0...0 ]
                              63               56..47

The bulk of KVM support for LAM is to emulate LAM's modified canonicality
checks.  The approach taken by KVM is to "fill" the metadata bits using the
highest bit of the translated address, e.g. for LAM-48, bit 47 is sign-extended
to bits 62:48.  The most significant bit, 63, is *not* modified, i.e. its value
from the raw, untagged virtual address is kept for the canonicality check. This
untagging allows

Aside from emulating LAM's canonical checks behavior, LAM has the usual KVM
touchpoints for selectable features: enumeration (CPUID.7.1:EAX.LAM[bit 26],
enabling via CR3 and CR4 bits, etc.

diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index f482216bbdb89719931c53b7a84a7aab09899941..378ed944b849fb0448a13bd7f12a7a542ab7e388 100644 (file)
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -137,6 +137,7 @@ KVM_X86_OP(msr_filter_changed)
 KVM_X86_OP(complete_emulated_msr)
 KVM_X86_OP(vcpu_deliver_sipi_vector)
 KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons);
+KVM_X86_OP_OPTIONAL(get_untagged_addr)
 
 #undef KVM_X86_OP
 #undef KVM_X86_OP_OPTIONAL

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c17e38ac8b7a36253337157a7b27b2c8b2d18302..c5e362e2116f5d0cf43dea830c0b21f935ac7999 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -133,7 +133,8 @@
                          | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
                          | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
                          | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
-                         | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
+                         | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \
+                         | X86_CR4_LAM_SUP))
 
 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
 
@@ -1790,6 +1791,8 @@ struct kvm_x86_ops {
         * Returns vCPU specific APICv inhibit reasons
         */
        unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu);
+
+       gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
 };
 
 struct kvm_x86_nested_ops {

diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 727b6551e1dfc0483bc3514e6ddfd52495f01b4d..294e5bd5f8a0ed06fcb482a8d7ffd3dc2ac6e636 100644 (file)
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -677,7 +677,7 @@ void kvm_set_cpu_caps(void)
        kvm_cpu_cap_mask(CPUID_7_1_EAX,
                F(AVX_VNNI) | F(AVX512_BF16) | F(CMPCCXADD) |
                F(FZRM) | F(FSRS) | F(FSRC) |
-               F(AMX_FP16) | F(AVX_IFMA)
+               F(AMX_FP16) | F(AVX_IFMA) | F(LAM)
        );
 
        kvm_cpu_cap_init_kvm_defined(CPUID_7_1_EDX,

diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index 0b90532b6e261430c7997e933f59f5531312d627..856e3037e74f3ffc7fdeb72f2067812080d71910 100644 (file)
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -47,11 +47,6 @@ static inline bool kvm_vcpu_is_legal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
        return !(gpa & vcpu->arch.reserved_gpa_bits);
 }
 
-static inline bool kvm_vcpu_is_illegal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
-{
-       return !kvm_vcpu_is_legal_gpa(vcpu, gpa);
-}
-
 static inline bool kvm_vcpu_is_legal_aligned_gpa(struct kvm_vcpu *vcpu,
                                                 gpa_t gpa, gpa_t alignment)
 {
@@ -279,4 +274,12 @@ static __always_inline bool guest_can_use(struct kvm_vcpu *vcpu,
                        vcpu->arch.governed_features.enabled);
 }
 
+static inline bool kvm_vcpu_is_legal_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
+{
+       if (guest_can_use(vcpu, X86_FEATURE_LAM))
+               cr3 &= ~(X86_CR3_LAM_U48 | X86_CR3_LAM_U57);
+
+       return kvm_vcpu_is_legal_gpa(vcpu, cr3);
+}
+
 #endif

diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 2673cd5c46cb486b9e78a20c589462552f0b4747..e223043ef5b26f23be5b2f0606641f66c5cd18aa 100644 (file)
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -687,8 +687,8 @@ static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
                                       struct segmented_address addr,
                                       unsigned *max_size, unsigned size,
-                                      bool write, bool fetch,
-                                      enum x86emul_mode mode, ulong *linear)
+                                      enum x86emul_mode mode, ulong *linear,
+                                      unsigned int flags)
 {
        struct desc_struct desc;
        bool usable;
@@ -701,7 +701,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
        *max_size = 0;
        switch (mode) {
        case X86EMUL_MODE_PROT64:
-               *linear = la;
+               *linear = la = ctxt->ops->get_untagged_addr(ctxt, la, flags);
                va_bits = ctxt_virt_addr_bits(ctxt);
                if (!__is_canonical_address(la, va_bits))
                        goto bad;
@@ -717,11 +717,11 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
                if (!usable)
                        goto bad;
                /* code segment in protected mode or read-only data segment */
-               if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
-                                       || !(desc.type & 2)) && write)
+               if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) || !(desc.type & 2)) &&
+                   (flags & X86EMUL_F_WRITE))
                        goto bad;
                /* unreadable code segment */
-               if (!fetch && (desc.type & 8) && !(desc.type & 2))
+               if (!(flags & X86EMUL_F_FETCH) && (desc.type & 8) && !(desc.type & 2))
                        goto bad;
                lim = desc_limit_scaled(&desc);
                if (!(desc.type & 8) && (desc.type & 4)) {
@@ -757,8 +757,8 @@ static int linearize(struct x86_emulate_ctxt *ctxt,
                     ulong *linear)
 {
        unsigned max_size;
-       return __linearize(ctxt, addr, &max_size, size, write, false,
-                          ctxt->mode, linear);
+       return __linearize(ctxt, addr, &max_size, size, ctxt->mode, linear,
+                          write ? X86EMUL_F_WRITE : 0);
 }
 
 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)
@@ -771,7 +771,8 @@ static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)
 
        if (ctxt->op_bytes != sizeof(unsigned long))
                addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
-       rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);
+       rc = __linearize(ctxt, addr, &max_size, 1, ctxt->mode, &linear,
+                        X86EMUL_F_FETCH);
        if (rc == X86EMUL_CONTINUE)
                ctxt->_eip = addr.ea;
        return rc;
@@ -907,8 +908,8 @@ static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
         * boundary check itself.  Instead, we use max_size to check
         * against op_size.
         */
-       rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
-                        &linear);
+       rc = __linearize(ctxt, addr, &max_size, 0, ctxt->mode, &linear,
+                        X86EMUL_F_FETCH);
        if (unlikely(rc != X86EMUL_CONTINUE))
                return rc;
 
@@ -3439,8 +3440,10 @@ static int em_invlpg(struct x86_emulate_ctxt *ctxt)
 {
        int rc;
        ulong linear;
+       unsigned int max_size;
 
-       rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
+       rc = __linearize(ctxt, ctxt->src.addr.mem, &max_size, 1, ctxt->mode,
+                        &linear, X86EMUL_F_INVLPG);
        if (rc == X86EMUL_CONTINUE)
                ctxt->ops->invlpg(ctxt, linear);
        /* Disable writeback. */

diff --git a/arch/x86/kvm/governed_features.h b/arch/x86/kvm/governed_features.h
index 423a73395c102ca908453016e416dbdfb7fc2b5b..ad463b1ed4e4a87c29aa9d5af3842fbd4e039e41 100644 (file)
--- a/arch/x86/kvm/governed_features.h
+++ b/arch/x86/kvm/governed_features.h
@@ -16,6 +16,7 @@ KVM_GOVERNED_X86_FEATURE(PAUSEFILTER)
 KVM_GOVERNED_X86_FEATURE(PFTHRESHOLD)
 KVM_GOVERNED_X86_FEATURE(VGIF)
 KVM_GOVERNED_X86_FEATURE(VNMI)
+KVM_GOVERNED_X86_FEATURE(LAM)
 
 #undef KVM_GOVERNED_X86_FEATURE
 #undef KVM_GOVERNED_FEATURE

diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h
index be7aeb9b8ea3b152b269870e5a737642a492e192..e6d149825169dda3ace396ca979923c4a2d108e8 100644 (file)
--- a/arch/x86/kvm/kvm_emulate.h
+++ b/arch/x86/kvm/kvm_emulate.h
@@ -88,6 +88,12 @@ struct x86_instruction_info {
 #define X86EMUL_IO_NEEDED       5 /* IO is needed to complete emulation */
 #define X86EMUL_INTERCEPTED     6 /* Intercepted by nested VMCB/VMCS */
 
+/* x86-specific emulation flags */
+#define X86EMUL_F_WRITE                        BIT(0)
+#define X86EMUL_F_FETCH                        BIT(1)
+#define X86EMUL_F_IMPLICIT             BIT(2)
+#define X86EMUL_F_INVLPG               BIT(3)
+
 struct x86_emulate_ops {
        void (*vm_bugged)(struct x86_emulate_ctxt *ctxt);
        /*
@@ -224,6 +230,9 @@ struct x86_emulate_ops {
        int (*leave_smm)(struct x86_emulate_ctxt *ctxt);
        void (*triple_fault)(struct x86_emulate_ctxt *ctxt);
        int (*set_xcr)(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr);
+
+       gva_t (*get_untagged_addr)(struct x86_emulate_ctxt *ctxt, gva_t addr,
+                                  unsigned int flags);
 };
 
 /* Type, address-of, and value of an instruction's operand. */

diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index bb8c86eefac047de05f25c937fafd66f7e2ebe6a..60f21bb4c27b196d7da820a304771b31b700899f 100644 (file)
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -146,6 +146,14 @@ static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu)
        return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu));
 }
 
+static inline unsigned long kvm_get_active_cr3_lam_bits(struct kvm_vcpu *vcpu)
+{
+       if (!guest_can_use(vcpu, X86_FEATURE_LAM))
+               return 0;
+
+       return kvm_read_cr3(vcpu) & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57);
+}
+
 static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
 {
        u64 root_hpa = vcpu->arch.mmu->root.hpa;

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 8531480e5da4899ee7ef00e090e42e165c6f6e6d..d0590b417d3005cf28546ef6c380f3ee1e85d9e1 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -3802,7 +3802,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
        hpa_t root;
 
        root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);
-       root_gfn = root_pgd >> PAGE_SHIFT;
+       root_gfn = (root_pgd & __PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
 
        if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
                mmu->root.hpa = kvm_mmu_get_dummy_root();

diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h
index b66a7d47e0e4effbf62d67e69c500e838ef60170..0669a8a668cacd4d0be68affbecbb686524c5213 100644 (file)
--- a/arch/x86/kvm/mmu/mmu_internal.h
+++ b/arch/x86/kvm/mmu/mmu_internal.h
@@ -13,6 +13,7 @@
 #endif
 
 /* Page table builder macros common to shadow (host) PTEs and guest PTEs. */
+#define __PT_BASE_ADDR_MASK GENMASK_ULL(51, 12)
 #define __PT_LEVEL_SHIFT(level, bits_per_level)        \
        (PAGE_SHIFT + ((level) - 1) * (bits_per_level))
 #define __PT_INDEX(address, level, bits_per_level) \

diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index c85255073f67231f0d25417232c99c9eeceb8f7b..4d4e98fe4f3548baf9156f3a9e0fd67003df7fdf 100644 (file)
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -62,7 +62,7 @@
 #endif
 
 /* Common logic, but per-type values.  These also need to be undefined. */
-#define PT_BASE_ADDR_MASK      ((pt_element_t)(((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)))
+#define PT_BASE_ADDR_MASK      ((pt_element_t)__PT_BASE_ADDR_MASK)
 #define PT_LVL_ADDR_MASK(lvl)  __PT_LVL_ADDR_MASK(PT_BASE_ADDR_MASK, lvl, PT_LEVEL_BITS)
 #define PT_LVL_OFFSET_MASK(lvl)        __PT_LVL_OFFSET_MASK(PT_BASE_ADDR_MASK, lvl, PT_LEVEL_BITS)
 #define PT_INDEX(addr, lvl)    __PT_INDEX(addr, lvl, PT_LEVEL_BITS)

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 20212aac050b800534b2157d515e52b2721e217d..0cd34d011a973be77ae10d6856da0c41bde5d622 100644 (file)
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -317,7 +317,7 @@ static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu,
        if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) {
                if (CC(!(save->cr4 & X86_CR4_PAE)) ||
                    CC(!(save->cr0 & X86_CR0_PE)) ||
-                   CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3)))
+                   CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3)))
                        return false;
        }
 
@@ -522,7 +522,7 @@ static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu)
 static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
                               bool nested_npt, bool reload_pdptrs)
 {
-       if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3)))
+       if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3)))
                return -EINVAL;
 
        if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) &&

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index cf47b8b7f40f4f78365a7c1b3a68460bc97e318d..db0ad1e6ec4b0bc2dfa07c2328f4877efe86e474 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -1116,7 +1116,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
                               bool nested_ept, bool reload_pdptrs,
                               enum vm_entry_failure_code *entry_failure_code)
 {
-       if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) {
+       if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) {
                *entry_failure_code = ENTRY_FAIL_DEFAULT;
                return -EINVAL;
        }
@@ -2753,7 +2753,7 @@ static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)
        }
 
        /* Reserved bits should not be set */
-       if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f)))
+       if (CC(!kvm_vcpu_is_legal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f)))
                return false;
 
        /* AD, if set, should be supported */
@@ -2950,7 +2950,7 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
 
        if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) ||
            CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) ||
-           CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3)))
+           CC(!kvm_vcpu_is_legal_cr3(vcpu, vmcs12->host_cr3)))
                return -EINVAL;
 
        if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) ||
@@ -5026,6 +5026,7 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
                else
                        *ret = off;
 
+               *ret = vmx_get_untagged_addr(vcpu, *ret, 0);
                /* Long mode: #GP(0)/#SS(0) if the memory address is in a
                 * non-canonical form. This is the only check on the memory
                 * destination for long mode!
@@ -5830,6 +5831,10 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
        vpid02 = nested_get_vpid02(vcpu);
        switch (type) {
        case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
+               /*
+                * LAM doesn't apply to addresses that are inputs to TLB
+                * invalidation.
+                */
                if (!operand.vpid ||
                    is_noncanonical_address(operand.gla, vcpu))
                        return nested_vmx_fail(vcpu,

diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c
index 3e822e58249753c5ff267b5ce4f69e2ede9a00c9..6fef01e0536e5079c34e6efc6b64716119c10c4e 100644 (file)
--- a/arch/x86/kvm/vmx/sgx.c
+++ b/arch/x86/kvm/vmx/sgx.c
@@ -37,6 +37,7 @@ static int sgx_get_encls_gva(struct kvm_vcpu *vcpu, unsigned long offset,
        if (!IS_ALIGNED(*gva, alignment)) {
                fault = true;
        } else if (likely(is_64_bit_mode(vcpu))) {
+               *gva = vmx_get_untagged_addr(vcpu, *gva, 0);
                fault = is_noncanonical_address(*gva, vcpu);
        } else {
                *gva &= 0xffffffff;

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index a26603ddc968fe172702052a7793a94911c3b3f0..d21f55f323ea24000e115032338823879de9c997 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -3395,7 +3395,8 @@ static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
                        update_guest_cr3 = false;
                vmx_ept_load_pdptrs(vcpu);
        } else {
-               guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu);
+               guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu) |
+                           kvm_get_active_cr3_lam_bits(vcpu);
        }
 
        if (update_guest_cr3)
@@ -5780,7 +5781,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu)
         * would also use advanced VM-exit information for EPT violations to
         * reconstruct the page fault error code.
         */
-       if (unlikely(allow_smaller_maxphyaddr && kvm_vcpu_is_illegal_gpa(vcpu, gpa)))
+       if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa)))
                return kvm_emulate_instruction(vcpu, 0);
 
        return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
@@ -7671,6 +7672,9 @@ static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
        cr4_fixed1_update(X86_CR4_UMIP,       ecx, feature_bit(UMIP));
        cr4_fixed1_update(X86_CR4_LA57,       ecx, feature_bit(LA57));
 
+       entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 1);
+       cr4_fixed1_update(X86_CR4_LAM_SUP,    eax, feature_bit(LAM));
+
 #undef cr4_fixed1_update
 }
 
@@ -7757,6 +7761,7 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
                kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_XSAVES);
 
        kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VMX);
+       kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LAM);
 
        vmx_setup_uret_msrs(vmx);
 
@@ -8203,6 +8208,50 @@ static void vmx_vm_destroy(struct kvm *kvm)
        free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm));
 }
 
+/*
+ * Note, the SDM states that the linear address is masked *after* the modified
+ * canonicality check, whereas KVM masks (untags) the address and then performs
+ * a "normal" canonicality check.  Functionally, the two methods are identical,
+ * and when the masking occurs relative to the canonicality check isn't visible
+ * to software, i.e. KVM's behavior doesn't violate the SDM.
+ */
+gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags)
+{
+       int lam_bit;
+       unsigned long cr3_bits;
+
+       if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG))
+               return gva;
+
+       if (!is_64_bit_mode(vcpu))
+               return gva;
+
+       /*
+        * Bit 63 determines if the address should be treated as user address
+        * or a supervisor address.
+        */
+       if (!(gva & BIT_ULL(63))) {
+               cr3_bits = kvm_get_active_cr3_lam_bits(vcpu);
+               if (!(cr3_bits & (X86_CR3_LAM_U57 | X86_CR3_LAM_U48)))
+                       return gva;
+
+               /* LAM_U48 is ignored if LAM_U57 is set. */
+               lam_bit = cr3_bits & X86_CR3_LAM_U57 ? 56 : 47;
+       } else {
+               if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP))
+                       return gva;
+
+               lam_bit = kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 56 : 47;
+       }
+
+       /*
+        * Untag the address by sign-extending the lam_bit, but NOT to bit 63.
+        * Bit 63 is retained from the raw virtual address so that untagging
+        * doesn't change a user access to a supervisor access, and vice versa.
+        */
+       return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63));
+}
+
 static struct kvm_x86_ops vmx_x86_ops __initdata = {
        .name = KBUILD_MODNAME,
 
@@ -8343,6 +8392,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
        .complete_emulated_msr = kvm_complete_insn_gp,
 
        .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
+
+       .get_untagged_addr = vmx_get_untagged_addr,
 };
 
 static unsigned int vmx_handle_intel_pt_intr(void)

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index 8fe6eb2b4a34c41225ad55fc611dc3776fee9ef7..e3b0985bb74a1f4d57be41cbb0d283abbc476625 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -422,6 +422,8 @@ void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type);
 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu);
 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
 
+gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
+
 static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr,
                                             int type, bool value)
 {

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 242a8417ef227a4867ebd2b7ae6479d28c8c4a94..0366eca119d70f1089ff27e4bbf00a23aa04f090 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1284,7 +1284,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
         * the current vCPU mode is accurate.
         */
-       if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
+       if (!kvm_vcpu_is_legal_cr3(vcpu, cr3))
                return 1;
 
        if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
@@ -8471,6 +8471,15 @@ static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt)
                kvm_vm_bugged(kvm);
 }
 
+static gva_t emulator_get_untagged_addr(struct x86_emulate_ctxt *ctxt,
+                                       gva_t addr, unsigned int flags)
+{
+       if (!kvm_x86_ops.get_untagged_addr)
+               return addr;
+
+       return static_call(kvm_x86_get_untagged_addr)(emul_to_vcpu(ctxt), addr, flags);
+}
+
 static const struct x86_emulate_ops emulate_ops = {
        .vm_bugged           = emulator_vm_bugged,
        .read_gpr            = emulator_read_gpr,
@@ -8515,6 +8524,7 @@ static const struct x86_emulate_ops emulate_ops = {
        .leave_smm           = emulator_leave_smm,
        .triple_fault        = emulator_triple_fault,
        .set_xcr             = emulator_set_xcr,
+       .get_untagged_addr   = emulator_get_untagged_addr,
 };
 
 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
@@ -11643,7 +11653,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
                 */
                if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
                        return false;
-               if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
+               if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3))
                        return false;
        } else {
                /*
@@ -13588,6 +13598,10 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
 
        switch (type) {
        case INVPCID_TYPE_INDIV_ADDR:
+               /*
+                * LAM doesn't apply to addresses that are inputs to TLB
+                * invalidation.
+                */
                if ((!pcid_enabled && (operand.pcid != 0)) ||
                    is_noncanonical_address(operand.gla, vcpu)) {
                        kvm_inject_gp(vcpu, 0);

diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 5184fde1dc541a90ca150f16d71eb2bd9506b4af..2f7e191666580085c85785ada86789fb9d1842b1 100644 (file)
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -530,6 +530,8 @@ bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type);
                __reserved_bits |= X86_CR4_VMXE;        \
        if (!__cpu_has(__c, X86_FEATURE_PCID))          \
                __reserved_bits |= X86_CR4_PCIDE;       \
+       if (!__cpu_has(__c, X86_FEATURE_LAM))           \
+               __reserved_bits |= X86_CR4_LAM_SUP;     \
        __reserved_bits;                                \
 })