RISC-V: KVM: Cache gstage pgd_levels in struct kvm_gstage

Gstage page-table helpers frequently chase gstage->kvm->arch to
fetch pgd_levels. This adds noise and repeats the same dereference
chain in hot paths.

Add pgd_levels to struct kvm_gstage and initialize it from kvm->arch
when setting up a gstage instance. Introduce kvm_riscv_gstage_init()
to centralize initialization and switch gstage code to use
gstage->pgd_levels.

Suggested-by: Anup Patel <anup@brainfault.org>
Signed-off-by: Fangyu Yu <fangyu.yu@linux.alibaba.com>
Reviewed-by: Anup Patel <anup@brainfault.org>
Reviewed-by: Nutty Liu <nutty.liu@hotmail.com>
Link: https://lore.kernel.org/r/20260403153019.9916-3-fangyu.yu@linux.alibaba.com
Signed-off-by: Anup Patel <anup@brainfault.org>

diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h
index c35874768641c0b6098c1a7e9afb92125382e86b..9c908432bc173b80c766fe89b051debc7c185463 100644 (file)
--- a/arch/riscv/include/asm/kvm_gstage.h
+++ b/arch/riscv/include/asm/kvm_gstage.h
@@ -15,6 +15,7 @@ struct kvm_gstage {
 #define KVM_GSTAGE_FLAGS_LOCAL         BIT(0)
        unsigned long vmid;
        pgd_t *pgd;
+       unsigned long pgd_levels;
 };
 
 struct kvm_gstage_mapping {
@@ -96,4 +97,13 @@ static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels)
        }
 }
 
+static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm *kvm)
+{
+       gstage->kvm = kvm;
+       gstage->flags = 0;
+       gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid);
+       gstage->pgd = kvm->arch.pgd;
+       gstage->pgd_levels = kvm->arch.pgd_levels;
+}
+
 #endif

diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c
index bf7e54af2c7cf9fc2be046fba2ed8722633d8df5..d9fe8be2a15169318d5c9cc29fcfeaf9e084eb1c 100644 (file)
--- a/arch/riscv/kvm/gstage.c
+++ b/arch/riscv/kvm/gstage.c
@@ -26,7 +26,7 @@ static inline unsigned long gstage_pte_index(struct kvm_gstage *gstage,
        unsigned long mask;
        unsigned long shift = HGATP_PAGE_SHIFT + (kvm_riscv_gstage_index_bits * level);
 
-       if (level == gstage->kvm->arch.pgd_levels - 1)
+       if (level == gstage->pgd_levels - 1)
                mask = (PTRS_PER_PTE * (1UL << kvm_riscv_gstage_pgd_xbits)) - 1;
        else
                mask = PTRS_PER_PTE - 1;
@@ -45,7 +45,7 @@ static int gstage_page_size_to_level(struct kvm_gstage *gstage, unsigned long pa
        u32 i;
        unsigned long psz = 1UL << 12;
 
-       for (i = 0; i < gstage->kvm->arch.pgd_levels; i++) {
+       for (i = 0; i < gstage->pgd_levels; i++) {
                if (page_size == (psz << (i * kvm_riscv_gstage_index_bits))) {
                        *out_level = i;
                        return 0;
@@ -58,7 +58,7 @@ static int gstage_page_size_to_level(struct kvm_gstage *gstage, unsigned long pa
 static int gstage_level_to_page_order(struct kvm_gstage *gstage, u32 level,
                                      unsigned long *out_pgorder)
 {
-       if (gstage->kvm->arch.pgd_levels < level)
+       if (gstage->pgd_levels < level)
                return -EINVAL;
 
        *out_pgorder = 12 + (level * kvm_riscv_gstage_index_bits);
@@ -83,7 +83,7 @@ bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,
                               pte_t **ptepp, u32 *ptep_level)
 {
        pte_t *ptep;
-       u32 current_level = gstage->kvm->arch.pgd_levels - 1;
+       u32 current_level = gstage->pgd_levels - 1;
 
        *ptep_level = current_level;
        ptep = (pte_t *)gstage->pgd;
@@ -127,7 +127,7 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,
                             struct kvm_mmu_memory_cache *pcache,
                             const struct kvm_gstage_mapping *map)
 {
-       u32 current_level = gstage->kvm->arch.pgd_levels - 1;
+       u32 current_level = gstage->pgd_levels - 1;
        pte_t *next_ptep = (pte_t *)gstage->pgd;
        pte_t *ptep = &next_ptep[gstage_pte_index(gstage, map->addr, current_level)];
 
@@ -288,7 +288,7 @@ int kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage,
                                struct kvm_mmu_memory_cache *pcache,
                                gpa_t addr, u32 target_level, bool flush)
 {
-       u32 current_level = gstage->kvm->arch.pgd_levels - 1;
+       u32 current_level = gstage->pgd_levels - 1;
        pte_t *next_ptep = (pte_t *)gstage->pgd;
        unsigned long huge_pte, child_pte;
        unsigned long child_page_size;

diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index fbcdd75cb9af048a54073573ab646af6f67aa087..2d3def024270c0cf65621c984fd413cf1c7c8c3d 100644 (file)
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -24,10 +24,7 @@ static void mmu_wp_memory_region(struct kvm *kvm, int slot)
        phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
        struct kvm_gstage gstage;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
 
        spin_lock(&kvm->mmu_lock);
        kvm_riscv_gstage_wp_range(&gstage, start, end);
@@ -49,10 +46,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
        struct kvm_gstage_mapping map;
        struct kvm_gstage gstage;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
 
        end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK;
        pfn = __phys_to_pfn(hpa);
@@ -89,10 +83,7 @@ void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
 {
        struct kvm_gstage gstage;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
 
        spin_lock(&kvm->mmu_lock);
        kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
@@ -109,10 +100,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
        phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
        struct kvm_gstage gstage;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
 
        kvm_riscv_gstage_wp_range(&gstage, start, end);
 }
@@ -141,10 +129,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
        phys_addr_t size = slot->npages << PAGE_SHIFT;
        struct kvm_gstage gstage;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
 
        spin_lock(&kvm->mmu_lock);
        kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
@@ -250,10 +235,7 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
        if (!kvm->arch.pgd)
                return false;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
        mmu_locked = spin_trylock(&kvm->mmu_lock);
        kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT,
                                     (range->end - range->start) << PAGE_SHIFT,
@@ -275,10 +257,7 @@ bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
 
        WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
        if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
                                       &ptep, &ptep_level))
                return false;
@@ -298,10 +277,7 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
 
        WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
        if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
                                       &ptep, &ptep_level))
                return false;
@@ -463,10 +439,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
        struct kvm_gstage gstage;
        struct page *page;
 
-       gstage.kvm = kvm;
-       gstage.flags = 0;
-       gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-       gstage.pgd = kvm->arch.pgd;
+       kvm_riscv_gstage_init(&gstage, kvm);
 
        /* Setup initial state of output mapping */
        memset(out_map, 0, sizeof(*out_map));
@@ -587,10 +560,7 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
 
        spin_lock(&kvm->mmu_lock);
        if (kvm->arch.pgd) {
-               gstage.kvm = kvm;
-               gstage.flags = 0;
-               gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
-               gstage.pgd = kvm->arch.pgd;
+               kvm_riscv_gstage_init(&gstage, kvm);
                kvm_riscv_gstage_unmap_range(&gstage, 0UL,
                        kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels), false);
                pgd = READ_ONCE(kvm->arch.pgd);