--- /dev/null
+From stable+bounces-75645-greg=kroah.com@vger.kernel.org Tue Sep 10 15:12:27 2024
+From: He Lugang <helugang@uniontech.com>
+Date: Tue, 10 Sep 2024 21:11:18 +0800
+Subject: LoongArch: KVM: Implement kvm mmu operations
+To: stable@vger.kernel.org
+Cc: Tianrui Zhao <zhaotianrui@loongson.cn>, Bibo Mao <maobibo@loongson.cn>, Huacai Chen <chenhuacai@loongson.cn>, He Lugang <helugang@uniontech.com>
+Message-ID: <091CB73198EC428B+20240910131119.18625-1-helugang@uniontech.com>
+
+From: Tianrui Zhao <zhaotianrui@loongson.cn>
+
+commit 752e2cd7b4fb412f3e008493e0195e357bab9773 upstream
+
+Implement LoongArch kvm mmu, it is used to switch gpa to hpa when guest
+exit because of address translation exception.
+
+This patch implement: allocating gpa page table, searching gpa from it,
+and flushing guest gpa in the table.
+
+Reviewed-by: Bibo Mao <maobibo@loongson.cn>
+Tested-by: Huacai Chen <chenhuacai@loongson.cn>
+Signed-off-by: Tianrui Zhao <zhaotianrui@loongson.cn>
+Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
+Signed-off-by: He Lugang <helugang@uniontech.com>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/loongarch/include/asm/kvm_mmu.h | 139 +++++
+ arch/loongarch/kvm/mmu.c | 914 +++++++++++++++++++++++++++++++++++
+ 2 files changed, 1053 insertions(+)
+ create mode 100644 arch/loongarch/include/asm/kvm_mmu.h
+ create mode 100644 arch/loongarch/kvm/mmu.c
+
+--- /dev/null
++++ b/arch/loongarch/include/asm/kvm_mmu.h
+@@ -0,0 +1,139 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
++ */
++
++#ifndef __ASM_LOONGARCH_KVM_MMU_H__
++#define __ASM_LOONGARCH_KVM_MMU_H__
++
++#include <linux/kvm_host.h>
++#include <asm/pgalloc.h>
++#include <asm/tlb.h>
++
++/*
++ * KVM_MMU_CACHE_MIN_PAGES is the number of GPA page table translation levels
++ * for which pages need to be cached.
++ */
++#define KVM_MMU_CACHE_MIN_PAGES (CONFIG_PGTABLE_LEVELS - 1)
++
++#define _KVM_FLUSH_PGTABLE 0x1
++#define _KVM_HAS_PGMASK 0x2
++#define kvm_pfn_pte(pfn, prot) (((pfn) << PFN_PTE_SHIFT) | pgprot_val(prot))
++#define kvm_pte_pfn(x) ((phys_addr_t)((x & _PFN_MASK) >> PFN_PTE_SHIFT))
++
++typedef unsigned long kvm_pte_t;
++typedef struct kvm_ptw_ctx kvm_ptw_ctx;
++typedef int (*kvm_pte_ops)(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx);
++
++struct kvm_ptw_ctx {
++ kvm_pte_ops ops;
++ unsigned long flag;
++
++ /* for kvm_arch_mmu_enable_log_dirty_pt_masked use */
++ unsigned long mask;
++ unsigned long gfn;
++
++ /* page walk mmu info */
++ unsigned int level;
++ unsigned long pgtable_shift;
++ unsigned long invalid_entry;
++ unsigned long *invalid_ptes;
++ unsigned int *pte_shifts;
++ void *opaque;
++
++ /* free pte table page list */
++ struct list_head list;
++};
++
++kvm_pte_t *kvm_pgd_alloc(void);
++
++static inline void kvm_set_pte(kvm_pte_t *ptep, kvm_pte_t val)
++{
++ WRITE_ONCE(*ptep, val);
++}
++
++static inline int kvm_pte_write(kvm_pte_t pte) { return pte & _PAGE_WRITE; }
++static inline int kvm_pte_dirty(kvm_pte_t pte) { return pte & _PAGE_DIRTY; }
++static inline int kvm_pte_young(kvm_pte_t pte) { return pte & _PAGE_ACCESSED; }
++static inline int kvm_pte_huge(kvm_pte_t pte) { return pte & _PAGE_HUGE; }
++
++static inline kvm_pte_t kvm_pte_mkyoung(kvm_pte_t pte)
++{
++ return pte | _PAGE_ACCESSED;
++}
++
++static inline kvm_pte_t kvm_pte_mkold(kvm_pte_t pte)
++{
++ return pte & ~_PAGE_ACCESSED;
++}
++
++static inline kvm_pte_t kvm_pte_mkdirty(kvm_pte_t pte)
++{
++ return pte | _PAGE_DIRTY;
++}
++
++static inline kvm_pte_t kvm_pte_mkclean(kvm_pte_t pte)
++{
++ return pte & ~_PAGE_DIRTY;
++}
++
++static inline kvm_pte_t kvm_pte_mkhuge(kvm_pte_t pte)
++{
++ return pte | _PAGE_HUGE;
++}
++
++static inline kvm_pte_t kvm_pte_mksmall(kvm_pte_t pte)
++{
++ return pte & ~_PAGE_HUGE;
++}
++
++static inline int kvm_need_flush(kvm_ptw_ctx *ctx)
++{
++ return ctx->flag & _KVM_FLUSH_PGTABLE;
++}
++
++static inline kvm_pte_t *kvm_pgtable_offset(kvm_ptw_ctx *ctx, kvm_pte_t *table,
++ phys_addr_t addr)
++{
++
++ return table + ((addr >> ctx->pgtable_shift) & (PTRS_PER_PTE - 1));
++}
++
++static inline phys_addr_t kvm_pgtable_addr_end(kvm_ptw_ctx *ctx,
++ phys_addr_t addr, phys_addr_t end)
++{
++ phys_addr_t boundary, size;
++
++ size = 0x1UL << ctx->pgtable_shift;
++ boundary = (addr + size) & ~(size - 1);
++ return (boundary - 1 < end - 1) ? boundary : end;
++}
++
++static inline int kvm_pte_present(kvm_ptw_ctx *ctx, kvm_pte_t *entry)
++{
++ if (!ctx || ctx->level == 0)
++ return !!(*entry & _PAGE_PRESENT);
++
++ return *entry != ctx->invalid_entry;
++}
++
++static inline int kvm_pte_none(kvm_ptw_ctx *ctx, kvm_pte_t *entry)
++{
++ return *entry == ctx->invalid_entry;
++}
++
++static inline void kvm_ptw_enter(kvm_ptw_ctx *ctx)
++{
++ ctx->level--;
++ ctx->pgtable_shift = ctx->pte_shifts[ctx->level];
++ ctx->invalid_entry = ctx->invalid_ptes[ctx->level];
++}
++
++static inline void kvm_ptw_exit(kvm_ptw_ctx *ctx)
++{
++ ctx->level++;
++ ctx->pgtable_shift = ctx->pte_shifts[ctx->level];
++ ctx->invalid_entry = ctx->invalid_ptes[ctx->level];
++}
++
++#endif /* __ASM_LOONGARCH_KVM_MMU_H__ */
+--- /dev/null
++++ b/arch/loongarch/kvm/mmu.c
+@@ -0,0 +1,914 @@
++// SPDX-License-Identifier: GPL-2.0
++/*
++ * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
++ */
++
++#include <linux/highmem.h>
++#include <linux/hugetlb.h>
++#include <linux/kvm_host.h>
++#include <linux/page-flags.h>
++#include <linux/uaccess.h>
++#include <asm/mmu_context.h>
++#include <asm/pgalloc.h>
++#include <asm/tlb.h>
++#include <asm/kvm_mmu.h>
++
++static inline void kvm_ptw_prepare(struct kvm *kvm, kvm_ptw_ctx *ctx)
++{
++ ctx->level = kvm->arch.root_level;
++ /* pte table */
++ ctx->invalid_ptes = kvm->arch.invalid_ptes;
++ ctx->pte_shifts = kvm->arch.pte_shifts;
++ ctx->pgtable_shift = ctx->pte_shifts[ctx->level];
++ ctx->invalid_entry = ctx->invalid_ptes[ctx->level];
++ ctx->opaque = kvm;
++}
++
++/*
++ * Mark a range of guest physical address space old (all accesses fault) in the
++ * VM's GPA page table to allow detection of commonly used pages.
++ */
++static int kvm_mkold_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
++{
++ if (kvm_pte_young(*pte)) {
++ *pte = kvm_pte_mkold(*pte);
++ return 1;
++ }
++
++ return 0;
++}
++
++/*
++ * Mark a range of guest physical address space clean (writes fault) in the VM's
++ * GPA page table to allow dirty page tracking.
++ */
++static int kvm_mkclean_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
++{
++ gfn_t offset;
++ kvm_pte_t val;
++
++ val = *pte;
++ /*
++ * For kvm_arch_mmu_enable_log_dirty_pt_masked with mask, start and end
++ * may cross hugepage, for first huge page parameter addr is equal to
++ * start, however for the second huge page addr is base address of
++ * this huge page, rather than start or end address
++ */
++ if ((ctx->flag & _KVM_HAS_PGMASK) && !kvm_pte_huge(val)) {
++ offset = (addr >> PAGE_SHIFT) - ctx->gfn;
++ if (!(BIT(offset) & ctx->mask))
++ return 0;
++ }
++
++ /*
++ * Need not split huge page now, just set write-proect pte bit
++ * Split huge page until next write fault
++ */
++ if (kvm_pte_dirty(val)) {
++ *pte = kvm_pte_mkclean(val);
++ return 1;
++ }
++
++ return 0;
++}
++
++/*
++ * Clear pte entry
++ */
++static int kvm_flush_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
++{
++ struct kvm *kvm;
++
++ kvm = ctx->opaque;
++ if (ctx->level)
++ kvm->stat.hugepages--;
++ else
++ kvm->stat.pages--;
++
++ *pte = ctx->invalid_entry;
++
++ return 1;
++}
++
++/*
++ * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory.
++ *
++ * Allocate a blank KVM GPA page directory (PGD) for representing guest physical
++ * to host physical page mappings.
++ *
++ * Returns: Pointer to new KVM GPA page directory.
++ * NULL on allocation failure.
++ */
++kvm_pte_t *kvm_pgd_alloc(void)
++{
++ kvm_pte_t *pgd;
++
++ pgd = (kvm_pte_t *)__get_free_pages(GFP_KERNEL, 0);
++ if (pgd)
++ pgd_init((void *)pgd);
++
++ return pgd;
++}
++
++static void _kvm_pte_init(void *addr, unsigned long val)
++{
++ unsigned long *p, *end;
++
++ p = (unsigned long *)addr;
++ end = p + PTRS_PER_PTE;
++ do {
++ p[0] = val;
++ p[1] = val;
++ p[2] = val;
++ p[3] = val;
++ p[4] = val;
++ p += 8;
++ p[-3] = val;
++ p[-2] = val;
++ p[-1] = val;
++ } while (p != end);
++}
++
++/*
++ * Caller must hold kvm->mm_lock
++ *
++ * Walk the page tables of kvm to find the PTE corresponding to the
++ * address @addr. If page tables don't exist for @addr, they will be created
++ * from the MMU cache if @cache is not NULL.
++ */
++static kvm_pte_t *kvm_populate_gpa(struct kvm *kvm,
++ struct kvm_mmu_memory_cache *cache,
++ unsigned long addr, int level)
++{
++ kvm_ptw_ctx ctx;
++ kvm_pte_t *entry, *child;
++
++ kvm_ptw_prepare(kvm, &ctx);
++ child = kvm->arch.pgd;
++ while (ctx.level > level) {
++ entry = kvm_pgtable_offset(&ctx, child, addr);
++ if (kvm_pte_none(&ctx, entry)) {
++ if (!cache)
++ return NULL;
++
++ child = kvm_mmu_memory_cache_alloc(cache);
++ _kvm_pte_init(child, ctx.invalid_ptes[ctx.level - 1]);
++ kvm_set_pte(entry, __pa(child));
++ } else if (kvm_pte_huge(*entry)) {
++ return entry;
++ } else
++ child = (kvm_pte_t *)__va(PHYSADDR(*entry));
++ kvm_ptw_enter(&ctx);
++ }
++
++ entry = kvm_pgtable_offset(&ctx, child, addr);
++
++ return entry;
++}
++
++/*
++ * Page walker for VM shadow mmu at last level
++ * The last level is small pte page or huge pmd page
++ */
++static int kvm_ptw_leaf(kvm_pte_t *dir, phys_addr_t addr, phys_addr_t end, kvm_ptw_ctx *ctx)
++{
++ int ret;
++ phys_addr_t next, start, size;
++ struct list_head *list;
++ kvm_pte_t *entry, *child;
++
++ ret = 0;
++ start = addr;
++ child = (kvm_pte_t *)__va(PHYSADDR(*dir));
++ entry = kvm_pgtable_offset(ctx, child, addr);
++ do {
++ next = addr + (0x1UL << ctx->pgtable_shift);
++ if (!kvm_pte_present(ctx, entry))
++ continue;
++
++ ret |= ctx->ops(entry, addr, ctx);
++ } while (entry++, addr = next, addr < end);
++
++ if (kvm_need_flush(ctx)) {
++ size = 0x1UL << (ctx->pgtable_shift + PAGE_SHIFT - 3);
++ if (start + size == end) {
++ list = (struct list_head *)child;
++ list_add_tail(list, &ctx->list);
++ *dir = ctx->invalid_ptes[ctx->level + 1];
++ }
++ }
++
++ return ret;
++}
++
++/*
++ * Page walker for VM shadow mmu at page table dir level
++ */
++static int kvm_ptw_dir(kvm_pte_t *dir, phys_addr_t addr, phys_addr_t end, kvm_ptw_ctx *ctx)
++{
++ int ret;
++ phys_addr_t next, start, size;
++ struct list_head *list;
++ kvm_pte_t *entry, *child;
++
++ ret = 0;
++ start = addr;
++ child = (kvm_pte_t *)__va(PHYSADDR(*dir));
++ entry = kvm_pgtable_offset(ctx, child, addr);
++ do {
++ next = kvm_pgtable_addr_end(ctx, addr, end);
++ if (!kvm_pte_present(ctx, entry))
++ continue;
++
++ if (kvm_pte_huge(*entry)) {
++ ret |= ctx->ops(entry, addr, ctx);
++ continue;
++ }
++
++ kvm_ptw_enter(ctx);
++ if (ctx->level == 0)
++ ret |= kvm_ptw_leaf(entry, addr, next, ctx);
++ else
++ ret |= kvm_ptw_dir(entry, addr, next, ctx);
++ kvm_ptw_exit(ctx);
++ } while (entry++, addr = next, addr < end);
++
++ if (kvm_need_flush(ctx)) {
++ size = 0x1UL << (ctx->pgtable_shift + PAGE_SHIFT - 3);
++ if (start + size == end) {
++ list = (struct list_head *)child;
++ list_add_tail(list, &ctx->list);
++ *dir = ctx->invalid_ptes[ctx->level + 1];
++ }
++ }
++
++ return ret;
++}
++
++/*
++ * Page walker for VM shadow mmu at page root table
++ */
++static int kvm_ptw_top(kvm_pte_t *dir, phys_addr_t addr, phys_addr_t end, kvm_ptw_ctx *ctx)
++{
++ int ret;
++ phys_addr_t next;
++ kvm_pte_t *entry;
++
++ ret = 0;
++ entry = kvm_pgtable_offset(ctx, dir, addr);
++ do {
++ next = kvm_pgtable_addr_end(ctx, addr, end);
++ if (!kvm_pte_present(ctx, entry))
++ continue;
++
++ kvm_ptw_enter(ctx);
++ ret |= kvm_ptw_dir(entry, addr, next, ctx);
++ kvm_ptw_exit(ctx);
++ } while (entry++, addr = next, addr < end);
++
++ return ret;
++}
++
++/*
++ * kvm_flush_range() - Flush a range of guest physical addresses.
++ * @kvm: KVM pointer.
++ * @start_gfn: Guest frame number of first page in GPA range to flush.
++ * @end_gfn: Guest frame number of last page in GPA range to flush.
++ * @lock: Whether to hold mmu_lock or not
++ *
++ * Flushes a range of GPA mappings from the GPA page tables.
++ */
++static void kvm_flush_range(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn, int lock)
++{
++ int ret;
++ kvm_ptw_ctx ctx;
++ struct list_head *pos, *temp;
++
++ ctx.ops = kvm_flush_pte;
++ ctx.flag = _KVM_FLUSH_PGTABLE;
++ kvm_ptw_prepare(kvm, &ctx);
++ INIT_LIST_HEAD(&ctx.list);
++
++ if (lock) {
++ spin_lock(&kvm->mmu_lock);
++ ret = kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT,
++ end_gfn << PAGE_SHIFT, &ctx);
++ spin_unlock(&kvm->mmu_lock);
++ } else
++ ret = kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT,
++ end_gfn << PAGE_SHIFT, &ctx);
++
++ /* Flush vpid for each vCPU individually */
++ if (ret)
++ kvm_flush_remote_tlbs(kvm);
++
++ /*
++ * free pte table page after mmu_lock
++ * the pte table page is linked together with ctx.list
++ */
++ list_for_each_safe(pos, temp, &ctx.list) {
++ list_del(pos);
++ free_page((unsigned long)pos);
++ }
++}
++
++/*
++ * kvm_mkclean_gpa_pt() - Make a range of guest physical addresses clean.
++ * @kvm: KVM pointer.
++ * @start_gfn: Guest frame number of first page in GPA range to flush.
++ * @end_gfn: Guest frame number of last page in GPA range to flush.
++ *
++ * Make a range of GPA mappings clean so that guest writes will fault and
++ * trigger dirty page logging.
++ *
++ * The caller must hold the @kvm->mmu_lock spinlock.
++ *
++ * Returns: Whether any GPA mappings were modified, which would require
++ * derived mappings (GVA page tables & TLB enties) to be
++ * invalidated.
++ */
++static int kvm_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
++{
++ kvm_ptw_ctx ctx;
++
++ ctx.ops = kvm_mkclean_pte;
++ ctx.flag = 0;
++ kvm_ptw_prepare(kvm, &ctx);
++ return kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT, end_gfn << PAGE_SHIFT, &ctx);
++}
++
++/*
++ * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages
++ * @kvm: The KVM pointer
++ * @slot: The memory slot associated with mask
++ * @gfn_offset: The gfn offset in memory slot
++ * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory
++ * slot to be write protected
++ *
++ * Walks bits set in mask write protects the associated pte's. Caller must
++ * acquire @kvm->mmu_lock.
++ */
++void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
++ struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask)
++{
++ kvm_ptw_ctx ctx;
++ gfn_t base_gfn = slot->base_gfn + gfn_offset;
++ gfn_t start = base_gfn + __ffs(mask);
++ gfn_t end = base_gfn + __fls(mask) + 1;
++
++ ctx.ops = kvm_mkclean_pte;
++ ctx.flag = _KVM_HAS_PGMASK;
++ ctx.mask = mask;
++ ctx.gfn = base_gfn;
++ kvm_ptw_prepare(kvm, &ctx);
++
++ kvm_ptw_top(kvm->arch.pgd, start << PAGE_SHIFT, end << PAGE_SHIFT, &ctx);
++}
++
++void kvm_arch_commit_memory_region(struct kvm *kvm,
++ struct kvm_memory_slot *old,
++ const struct kvm_memory_slot *new,
++ enum kvm_mr_change change)
++{
++ int needs_flush;
++
++ /*
++ * If dirty page logging is enabled, write protect all pages in the slot
++ * ready for dirty logging.
++ *
++ * There is no need to do this in any of the following cases:
++ * CREATE: No dirty mappings will already exist.
++ * MOVE/DELETE: The old mappings will already have been cleaned up by
++ * kvm_arch_flush_shadow_memslot()
++ */
++ if (change == KVM_MR_FLAGS_ONLY &&
++ (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
++ new->flags & KVM_MEM_LOG_DIRTY_PAGES)) {
++ spin_lock(&kvm->mmu_lock);
++ /* Write protect GPA page table entries */
++ needs_flush = kvm_mkclean_gpa_pt(kvm, new->base_gfn,
++ new->base_gfn + new->npages);
++ spin_unlock(&kvm->mmu_lock);
++ if (needs_flush)
++ kvm_flush_remote_tlbs(kvm);
++ }
++}
++
++void kvm_arch_flush_shadow_all(struct kvm *kvm)
++{
++ kvm_flush_range(kvm, 0, kvm->arch.gpa_size >> PAGE_SHIFT, 0);
++}
++
++void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
++{
++ /*
++ * The slot has been made invalid (ready for moving or deletion), so we
++ * need to ensure that it can no longer be accessed by any guest vCPUs.
++ */
++ kvm_flush_range(kvm, slot->base_gfn, slot->base_gfn + slot->npages, 1);
++}
++
++bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
++{
++ kvm_ptw_ctx ctx;
++
++ ctx.flag = 0;
++ ctx.ops = kvm_flush_pte;
++ kvm_ptw_prepare(kvm, &ctx);
++ INIT_LIST_HEAD(&ctx.list);
++
++ return kvm_ptw_top(kvm->arch.pgd, range->start << PAGE_SHIFT,
++ range->end << PAGE_SHIFT, &ctx);
++}
++
++bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
++{
++ unsigned long prot_bits;
++ kvm_pte_t *ptep;
++ kvm_pfn_t pfn = pte_pfn(range->arg.pte);
++ gpa_t gpa = range->start << PAGE_SHIFT;
++
++ ptep = kvm_populate_gpa(kvm, NULL, gpa, 0);
++ if (!ptep)
++ return false;
++
++ /* Replacing an absent or old page doesn't need flushes */
++ if (!kvm_pte_present(NULL, ptep) || !kvm_pte_young(*ptep)) {
++ kvm_set_pte(ptep, 0);
++ return false;
++ }
++
++ /* Fill new pte if write protected or page migrated */
++ prot_bits = _PAGE_PRESENT | __READABLE;
++ prot_bits |= _CACHE_MASK & pte_val(range->arg.pte);
++
++ /*
++ * Set _PAGE_WRITE or _PAGE_DIRTY iff old and new pte both support
++ * _PAGE_WRITE for map_page_fast if next page write fault
++ * _PAGE_DIRTY since gpa has already recorded as dirty page
++ */
++ prot_bits |= __WRITEABLE & *ptep & pte_val(range->arg.pte);
++ kvm_set_pte(ptep, kvm_pfn_pte(pfn, __pgprot(prot_bits)));
++
++ return true;
++}
++
++bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
++{
++ kvm_ptw_ctx ctx;
++
++ ctx.flag = 0;
++ ctx.ops = kvm_mkold_pte;
++ kvm_ptw_prepare(kvm, &ctx);
++
++ return kvm_ptw_top(kvm->arch.pgd, range->start << PAGE_SHIFT,
++ range->end << PAGE_SHIFT, &ctx);
++}
++
++bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
++{
++ gpa_t gpa = range->start << PAGE_SHIFT;
++ kvm_pte_t *ptep = kvm_populate_gpa(kvm, NULL, gpa, 0);
++
++ if (ptep && kvm_pte_present(NULL, ptep) && kvm_pte_young(*ptep))
++ return true;
++
++ return false;
++}
++
++/*
++ * kvm_map_page_fast() - Fast path GPA fault handler.
++ * @vcpu: vCPU pointer.
++ * @gpa: Guest physical address of fault.
++ * @write: Whether the fault was due to a write.
++ *
++ * Perform fast path GPA fault handling, doing all that can be done without
++ * calling into KVM. This handles marking old pages young (for idle page
++ * tracking), and dirtying of clean pages (for dirty page logging).
++ *
++ * Returns: 0 on success, in which case we can update derived mappings and
++ * resume guest execution.
++ * -EFAULT on failure due to absent GPA mapping or write to
++ * read-only page, in which case KVM must be consulted.
++ */
++static int kvm_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa, bool write)
++{
++ int ret = 0;
++ kvm_pfn_t pfn = 0;
++ kvm_pte_t *ptep, changed, new;
++ gfn_t gfn = gpa >> PAGE_SHIFT;
++ struct kvm *kvm = vcpu->kvm;
++ struct kvm_memory_slot *slot;
++
++ spin_lock(&kvm->mmu_lock);
++
++ /* Fast path - just check GPA page table for an existing entry */
++ ptep = kvm_populate_gpa(kvm, NULL, gpa, 0);
++ if (!ptep || !kvm_pte_present(NULL, ptep)) {
++ ret = -EFAULT;
++ goto out;
++ }
++
++ /* Track access to pages marked old */
++ new = *ptep;
++ if (!kvm_pte_young(new))
++ new = kvm_pte_mkyoung(new);
++ /* call kvm_set_pfn_accessed() after unlock */
++
++ if (write && !kvm_pte_dirty(new)) {
++ if (!kvm_pte_write(new)) {
++ ret = -EFAULT;
++ goto out;
++ }
++
++ if (kvm_pte_huge(new)) {
++ /*
++ * Do not set write permission when dirty logging is
++ * enabled for HugePages
++ */
++ slot = gfn_to_memslot(kvm, gfn);
++ if (kvm_slot_dirty_track_enabled(slot)) {
++ ret = -EFAULT;
++ goto out;
++ }
++ }
++
++ /* Track dirtying of writeable pages */
++ new = kvm_pte_mkdirty(new);
++ }
++
++ changed = new ^ (*ptep);
++ if (changed) {
++ kvm_set_pte(ptep, new);
++ pfn = kvm_pte_pfn(new);
++ }
++ spin_unlock(&kvm->mmu_lock);
++
++ /*
++ * Fixme: pfn may be freed after mmu_lock
++ * kvm_try_get_pfn(pfn)/kvm_release_pfn pair to prevent this?
++ */
++ if (kvm_pte_young(changed))
++ kvm_set_pfn_accessed(pfn);
++
++ if (kvm_pte_dirty(changed)) {
++ mark_page_dirty(kvm, gfn);
++ kvm_set_pfn_dirty(pfn);
++ }
++ return ret;
++out:
++ spin_unlock(&kvm->mmu_lock);
++ return ret;
++}
++
++static bool fault_supports_huge_mapping(struct kvm_memory_slot *memslot,
++ unsigned long hva, unsigned long map_size, bool write)
++{
++ size_t size;
++ gpa_t gpa_start;
++ hva_t uaddr_start, uaddr_end;
++
++ /* Disable dirty logging on HugePages */
++ if (kvm_slot_dirty_track_enabled(memslot) && write)
++ return false;
++
++ size = memslot->npages * PAGE_SIZE;
++ gpa_start = memslot->base_gfn << PAGE_SHIFT;
++ uaddr_start = memslot->userspace_addr;
++ uaddr_end = uaddr_start + size;
++
++ /*
++ * Pages belonging to memslots that don't have the same alignment
++ * within a PMD for userspace and GPA cannot be mapped with stage-2
++ * PMD entries, because we'll end up mapping the wrong pages.
++ *
++ * Consider a layout like the following:
++ *
++ * memslot->userspace_addr:
++ * +-----+--------------------+--------------------+---+
++ * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz|
++ * +-----+--------------------+--------------------+---+
++ *
++ * memslot->base_gfn << PAGE_SIZE:
++ * +---+--------------------+--------------------+-----+
++ * |abc|def Stage-2 block | Stage-2 block |tvxyz|
++ * +---+--------------------+--------------------+-----+
++ *
++ * If we create those stage-2 blocks, we'll end up with this incorrect
++ * mapping:
++ * d -> f
++ * e -> g
++ * f -> h
++ */
++ if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1)))
++ return false;
++
++ /*
++ * Next, let's make sure we're not trying to map anything not covered
++ * by the memslot. This means we have to prohibit block size mappings
++ * for the beginning and end of a non-block aligned and non-block sized
++ * memory slot (illustrated by the head and tail parts of the
++ * userspace view above containing pages 'abcde' and 'xyz',
++ * respectively).
++ *
++ * Note that it doesn't matter if we do the check using the
++ * userspace_addr or the base_gfn, as both are equally aligned (per
++ * the check above) and equally sized.
++ */
++ return (hva & ~(map_size - 1)) >= uaddr_start &&
++ (hva & ~(map_size - 1)) + map_size <= uaddr_end;
++}
++
++/*
++ * Lookup the mapping level for @gfn in the current mm.
++ *
++ * WARNING! Use of host_pfn_mapping_level() requires the caller and the end
++ * consumer to be tied into KVM's handlers for MMU notifier events!
++ *
++ * There are several ways to safely use this helper:
++ *
++ * - Check mmu_invalidate_retry_hva() after grabbing the mapping level, before
++ * consuming it. In this case, mmu_lock doesn't need to be held during the
++ * lookup, but it does need to be held while checking the MMU notifier.
++ *
++ * - Hold mmu_lock AND ensure there is no in-progress MMU notifier invalidation
++ * event for the hva. This can be done by explicit checking the MMU notifier
++ * or by ensuring that KVM already has a valid mapping that covers the hva.
++ *
++ * - Do not use the result to install new mappings, e.g. use the host mapping
++ * level only to decide whether or not to zap an entry. In this case, it's
++ * not required to hold mmu_lock (though it's highly likely the caller will
++ * want to hold mmu_lock anyways, e.g. to modify SPTEs).
++ *
++ * Note! The lookup can still race with modifications to host page tables, but
++ * the above "rules" ensure KVM will not _consume_ the result of the walk if a
++ * race with the primary MMU occurs.
++ */
++static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn,
++ const struct kvm_memory_slot *slot)
++{
++ int level = 0;
++ unsigned long hva;
++ unsigned long flags;
++ pgd_t pgd;
++ p4d_t p4d;
++ pud_t pud;
++ pmd_t pmd;
++
++ /*
++ * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()
++ * is not solely for performance, it's also necessary to avoid the
++ * "writable" check in __gfn_to_hva_many(), which will always fail on
++ * read-only memslots due to gfn_to_hva() assuming writes. Earlier
++ * page fault steps have already verified the guest isn't writing a
++ * read-only memslot.
++ */
++ hva = __gfn_to_hva_memslot(slot, gfn);
++
++ /*
++ * Disable IRQs to prevent concurrent tear down of host page tables,
++ * e.g. if the primary MMU promotes a P*D to a huge page and then frees
++ * the original page table.
++ */
++ local_irq_save(flags);
++
++ /*
++ * Read each entry once. As above, a non-leaf entry can be promoted to
++ * a huge page _during_ this walk. Re-reading the entry could send the
++ * walk into the weeks, e.g. p*d_large() returns false (sees the old
++ * value) and then p*d_offset() walks into the target huge page instead
++ * of the old page table (sees the new value).
++ */
++ pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
++ if (pgd_none(pgd))
++ goto out;
++
++ p4d = READ_ONCE(*p4d_offset(&pgd, hva));
++ if (p4d_none(p4d) || !p4d_present(p4d))
++ goto out;
++
++ pud = READ_ONCE(*pud_offset(&p4d, hva));
++ if (pud_none(pud) || !pud_present(pud))
++ goto out;
++
++ pmd = READ_ONCE(*pmd_offset(&pud, hva));
++ if (pmd_none(pmd) || !pmd_present(pmd))
++ goto out;
++
++ if (kvm_pte_huge(pmd_val(pmd)))
++ level = 1;
++
++out:
++ local_irq_restore(flags);
++ return level;
++}
++
++/*
++ * Split huge page
++ */
++static kvm_pte_t *kvm_split_huge(struct kvm_vcpu *vcpu, kvm_pte_t *ptep, gfn_t gfn)
++{
++ int i;
++ kvm_pte_t val, *child;
++ struct kvm *kvm = vcpu->kvm;
++ struct kvm_mmu_memory_cache *memcache;
++
++ memcache = &vcpu->arch.mmu_page_cache;
++ child = kvm_mmu_memory_cache_alloc(memcache);
++ val = kvm_pte_mksmall(*ptep);
++ for (i = 0; i < PTRS_PER_PTE; i++) {
++ kvm_set_pte(child + i, val);
++ val += PAGE_SIZE;
++ }
++
++ /* The later kvm_flush_tlb_gpa() will flush hugepage tlb */
++ kvm_set_pte(ptep, __pa(child));
++
++ kvm->stat.hugepages--;
++ kvm->stat.pages += PTRS_PER_PTE;
++
++ return child + (gfn & (PTRS_PER_PTE - 1));
++}
++
++/*
++ * kvm_map_page() - Map a guest physical page.
++ * @vcpu: vCPU pointer.
++ * @gpa: Guest physical address of fault.
++ * @write: Whether the fault was due to a write.
++ *
++ * Handle GPA faults by creating a new GPA mapping (or updating an existing
++ * one).
++ *
++ * This takes care of marking pages young or dirty (idle/dirty page tracking),
++ * asking KVM for the corresponding PFN, and creating a mapping in the GPA page
++ * tables. Derived mappings (GVA page tables and TLBs) must be handled by the
++ * caller.
++ *
++ * Returns: 0 on success
++ * -EFAULT if there is no memory region at @gpa or a write was
++ * attempted to a read-only memory region. This is usually handled
++ * as an MMIO access.
++ */
++static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, bool write)
++{
++ bool writeable;
++ int srcu_idx, err, retry_no = 0, level;
++ unsigned long hva, mmu_seq, prot_bits;
++ kvm_pfn_t pfn;
++ kvm_pte_t *ptep, new_pte;
++ gfn_t gfn = gpa >> PAGE_SHIFT;
++ struct kvm *kvm = vcpu->kvm;
++ struct kvm_memory_slot *memslot;
++ struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
++
++ /* Try the fast path to handle old / clean pages */
++ srcu_idx = srcu_read_lock(&kvm->srcu);
++ err = kvm_map_page_fast(vcpu, gpa, write);
++ if (!err)
++ goto out;
++
++ memslot = gfn_to_memslot(kvm, gfn);
++ hva = gfn_to_hva_memslot_prot(memslot, gfn, &writeable);
++ if (kvm_is_error_hva(hva) || (write && !writeable)) {
++ err = -EFAULT;
++ goto out;
++ }
++
++ /* We need a minimum of cached pages ready for page table creation */
++ err = kvm_mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES);
++ if (err)
++ goto out;
++
++retry:
++ /*
++ * Used to check for invalidations in progress, of the pfn that is
++ * returned by pfn_to_pfn_prot below.
++ */
++ mmu_seq = kvm->mmu_invalidate_seq;
++ /*
++ * Ensure the read of mmu_invalidate_seq isn't reordered with PTE reads in
++ * gfn_to_pfn_prot() (which calls get_user_pages()), so that we don't
++ * risk the page we get a reference to getting unmapped before we have a
++ * chance to grab the mmu_lock without mmu_invalidate_retry() noticing.
++ *
++ * This smp_rmb() pairs with the effective smp_wmb() of the combination
++ * of the pte_unmap_unlock() after the PTE is zapped, and the
++ * spin_lock() in kvm_mmu_invalidate_invalidate_<page|range_end>() before
++ * mmu_invalidate_seq is incremented.
++ */
++ smp_rmb();
++
++ /* Slow path - ask KVM core whether we can access this GPA */
++ pfn = gfn_to_pfn_prot(kvm, gfn, write, &writeable);
++ if (is_error_noslot_pfn(pfn)) {
++ err = -EFAULT;
++ goto out;
++ }
++
++ /* Check if an invalidation has taken place since we got pfn */
++ spin_lock(&kvm->mmu_lock);
++ if (mmu_invalidate_retry_hva(kvm, mmu_seq, hva)) {
++ /*
++ * This can happen when mappings are changed asynchronously, but
++ * also synchronously if a COW is triggered by
++ * gfn_to_pfn_prot().
++ */
++ spin_unlock(&kvm->mmu_lock);
++ kvm_release_pfn_clean(pfn);
++ if (retry_no > 100) {
++ retry_no = 0;
++ schedule();
++ }
++ retry_no++;
++ goto retry;
++ }
++
++ /*
++ * For emulated devices such virtio device, actual cache attribute is
++ * determined by physical machine.
++ * For pass through physical device, it should be uncachable
++ */
++ prot_bits = _PAGE_PRESENT | __READABLE;
++ if (pfn_valid(pfn))
++ prot_bits |= _CACHE_CC;
++ else
++ prot_bits |= _CACHE_SUC;
++
++ if (writeable) {
++ prot_bits |= _PAGE_WRITE;
++ if (write)
++ prot_bits |= __WRITEABLE;
++ }
++
++ /* Disable dirty logging on HugePages */
++ level = 0;
++ if (!fault_supports_huge_mapping(memslot, hva, PMD_SIZE, write)) {
++ level = 0;
++ } else {
++ level = host_pfn_mapping_level(kvm, gfn, memslot);
++ if (level == 1) {
++ gfn = gfn & ~(PTRS_PER_PTE - 1);
++ pfn = pfn & ~(PTRS_PER_PTE - 1);
++ }
++ }
++
++ /* Ensure page tables are allocated */
++ ptep = kvm_populate_gpa(kvm, memcache, gpa, level);
++ new_pte = kvm_pfn_pte(pfn, __pgprot(prot_bits));
++ if (level == 1) {
++ new_pte = kvm_pte_mkhuge(new_pte);
++ /*
++ * previous pmd entry is invalid_pte_table
++ * there is invalid tlb with small page
++ * need flush these invalid tlbs for current vcpu
++ */
++ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
++ ++kvm->stat.hugepages;
++ } else if (kvm_pte_huge(*ptep) && write)
++ ptep = kvm_split_huge(vcpu, ptep, gfn);
++ else
++ ++kvm->stat.pages;
++ kvm_set_pte(ptep, new_pte);
++ spin_unlock(&kvm->mmu_lock);
++
++ if (prot_bits & _PAGE_DIRTY) {
++ mark_page_dirty_in_slot(kvm, memslot, gfn);
++ kvm_set_pfn_dirty(pfn);
++ }
++
++ kvm_set_pfn_accessed(pfn);
++ kvm_release_pfn_clean(pfn);
++out:
++ srcu_read_unlock(&kvm->srcu, srcu_idx);
++ return err;
++}
++
++int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write)
++{
++ int ret;
++
++ ret = kvm_map_page(vcpu, gpa, write);
++ if (ret)
++ return ret;
++
++ /* Invalidate this entry in the TLB */
++ kvm_flush_tlb_gpa(vcpu, gpa);
++
++ return 0;
++}
++
++void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
++{
++}
++
++int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old,
++ struct kvm_memory_slot *new, enum kvm_mr_change change)
++{
++ return 0;
++}
++
++void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
++ const struct kvm_memory_slot *memslot)
++{
++ kvm_flush_remote_tlbs(kvm);
++}
--- /dev/null
+From stable+bounces-75642-greg=kroah.com@vger.kernel.org Tue Sep 10 15:12:22 2024
+From: He Lugang <helugang@uniontech.com>
+Date: Tue, 10 Sep 2024 21:11:19 +0800
+Subject: LoongArch: Use accessors to page table entries instead of direct dereference
+To: stable@vger.kernel.org
+Cc: Huacai Chen <chenhuacai@loongson.cn>
+Message-ID: <1197B2966A66F7F9+20240910131119.18625-2-helugang@uniontech.com>
+
+From: Huacai Chen <chenhuacai@loongson.cn>
+
+commit 4574815abf43e2bf05643e1b3f7a2e5d6df894f0 upstream
+
+As very well explained in commit 20a004e7b017cce282 ("arm64: mm: Use
+READ_ONCE/WRITE_ONCE when accessing page tables"), an architecture whose
+page table walker can modify the PTE in parallel must use READ_ONCE()/
+WRITE_ONCE() macro to avoid any compiler transformation.
+
+So apply that to LoongArch which is such an architecture, in order to
+avoid potential problems.
+
+Similar to commit edf955647269422e ("riscv: Use accessors to page table
+entries instead of direct dereference").
+
+Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/loongarch/include/asm/hugetlb.h | 4 +-
+ arch/loongarch/include/asm/kfence.h | 6 ++--
+ arch/loongarch/include/asm/pgtable.h | 48 +++++++++++++++++++++--------------
+ arch/loongarch/kvm/mmu.c | 8 ++---
+ arch/loongarch/mm/hugetlbpage.c | 6 ++--
+ arch/loongarch/mm/init.c | 10 +++----
+ arch/loongarch/mm/kasan_init.c | 10 +++----
+ arch/loongarch/mm/pgtable.c | 2 -
+ 8 files changed, 52 insertions(+), 42 deletions(-)
+
+--- a/arch/loongarch/include/asm/hugetlb.h
++++ b/arch/loongarch/include/asm/hugetlb.h
+@@ -34,7 +34,7 @@ static inline pte_t huge_ptep_get_and_cl
+ unsigned long addr, pte_t *ptep)
+ {
+ pte_t clear;
+- pte_t pte = *ptep;
++ pte_t pte = ptep_get(ptep);
+
+ pte_val(clear) = (unsigned long)invalid_pte_table;
+ set_pte_at(mm, addr, ptep, clear);
+@@ -65,7 +65,7 @@ static inline int huge_ptep_set_access_f
+ pte_t *ptep, pte_t pte,
+ int dirty)
+ {
+- int changed = !pte_same(*ptep, pte);
++ int changed = !pte_same(ptep_get(ptep), pte);
+
+ if (changed) {
+ set_pte_at(vma->vm_mm, addr, ptep, pte);
+--- a/arch/loongarch/include/asm/kfence.h
++++ b/arch/loongarch/include/asm/kfence.h
+@@ -43,13 +43,13 @@ static inline bool kfence_protect_page(u
+ {
+ pte_t *pte = virt_to_kpte(addr);
+
+- if (WARN_ON(!pte) || pte_none(*pte))
++ if (WARN_ON(!pte) || pte_none(ptep_get(pte)))
+ return false;
+
+ if (protect)
+- set_pte(pte, __pte(pte_val(*pte) & ~(_PAGE_VALID | _PAGE_PRESENT)));
++ set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~(_PAGE_VALID | _PAGE_PRESENT)));
+ else
+- set_pte(pte, __pte(pte_val(*pte) | (_PAGE_VALID | _PAGE_PRESENT)));
++ set_pte(pte, __pte(pte_val(ptep_get(pte)) | (_PAGE_VALID | _PAGE_PRESENT)));
+
+ preempt_disable();
+ local_flush_tlb_one(addr);
+--- a/arch/loongarch/include/asm/pgtable.h
++++ b/arch/loongarch/include/asm/pgtable.h
+@@ -106,6 +106,9 @@ extern unsigned long empty_zero_page[PAG
+ #define KFENCE_AREA_START (VMEMMAP_END + 1)
+ #define KFENCE_AREA_END (KFENCE_AREA_START + KFENCE_AREA_SIZE - 1)
+
++#define ptep_get(ptep) READ_ONCE(*(ptep))
++#define pmdp_get(pmdp) READ_ONCE(*(pmdp))
++
+ #define pte_ERROR(e) \
+ pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
+ #ifndef __PAGETABLE_PMD_FOLDED
+@@ -147,11 +150,6 @@ static inline int p4d_present(p4d_t p4d)
+ return p4d_val(p4d) != (unsigned long)invalid_pud_table;
+ }
+
+-static inline void p4d_clear(p4d_t *p4dp)
+-{
+- p4d_val(*p4dp) = (unsigned long)invalid_pud_table;
+-}
+-
+ static inline pud_t *p4d_pgtable(p4d_t p4d)
+ {
+ return (pud_t *)p4d_val(p4d);
+@@ -159,7 +157,12 @@ static inline pud_t *p4d_pgtable(p4d_t p
+
+ static inline void set_p4d(p4d_t *p4d, p4d_t p4dval)
+ {
+- *p4d = p4dval;
++ WRITE_ONCE(*p4d, p4dval);
++}
++
++static inline void p4d_clear(p4d_t *p4dp)
++{
++ set_p4d(p4dp, __p4d((unsigned long)invalid_pud_table));
+ }
+
+ #define p4d_phys(p4d) PHYSADDR(p4d_val(p4d))
+@@ -193,17 +196,20 @@ static inline int pud_present(pud_t pud)
+ return pud_val(pud) != (unsigned long)invalid_pmd_table;
+ }
+
+-static inline void pud_clear(pud_t *pudp)
++static inline pmd_t *pud_pgtable(pud_t pud)
+ {
+- pud_val(*pudp) = ((unsigned long)invalid_pmd_table);
++ return (pmd_t *)pud_val(pud);
+ }
+
+-static inline pmd_t *pud_pgtable(pud_t pud)
++static inline void set_pud(pud_t *pud, pud_t pudval)
+ {
+- return (pmd_t *)pud_val(pud);
++ WRITE_ONCE(*pud, pudval);
+ }
+
+-#define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0)
++static inline void pud_clear(pud_t *pudp)
++{
++ set_pud(pudp, __pud((unsigned long)invalid_pmd_table));
++}
+
+ #define pud_phys(pud) PHYSADDR(pud_val(pud))
+ #define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT))
+@@ -231,12 +237,15 @@ static inline int pmd_present(pmd_t pmd)
+ return pmd_val(pmd) != (unsigned long)invalid_pte_table;
+ }
+
+-static inline void pmd_clear(pmd_t *pmdp)
++static inline void set_pmd(pmd_t *pmd, pmd_t pmdval)
+ {
+- pmd_val(*pmdp) = ((unsigned long)invalid_pte_table);
++ WRITE_ONCE(*pmd, pmdval);
+ }
+
+-#define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0)
++static inline void pmd_clear(pmd_t *pmdp)
++{
++ set_pmd(pmdp, __pmd((unsigned long)invalid_pte_table));
++}
+
+ #define pmd_phys(pmd) PHYSADDR(pmd_val(pmd))
+
+@@ -314,7 +323,8 @@ extern void paging_init(void);
+
+ static inline void set_pte(pte_t *ptep, pte_t pteval)
+ {
+- *ptep = pteval;
++ WRITE_ONCE(*ptep, pteval);
++
+ if (pte_val(pteval) & _PAGE_GLOBAL) {
+ pte_t *buddy = ptep_buddy(ptep);
+ /*
+@@ -341,8 +351,8 @@ static inline void set_pte(pte_t *ptep,
+ : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp)
+ : [global] "r" (page_global));
+ #else /* !CONFIG_SMP */
+- if (pte_none(*buddy))
+- pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL;
++ if (pte_none(ptep_get(buddy)))
++ WRITE_ONCE(*buddy, __pte(pte_val(ptep_get(buddy)) | _PAGE_GLOBAL));
+ #endif /* CONFIG_SMP */
+ }
+ }
+@@ -350,7 +360,7 @@ static inline void set_pte(pte_t *ptep,
+ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+ {
+ /* Preserve global status for the pair */
+- if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL)
++ if (pte_val(ptep_get(ptep_buddy(ptep))) & _PAGE_GLOBAL)
+ set_pte(ptep, __pte(_PAGE_GLOBAL));
+ else
+ set_pte(ptep, __pte(0));
+@@ -591,7 +601,7 @@ static inline pmd_t pmd_mkinvalid(pmd_t
+ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
+ unsigned long address, pmd_t *pmdp)
+ {
+- pmd_t old = *pmdp;
++ pmd_t old = pmdp_get(pmdp);
+
+ pmd_clear(pmdp);
+
+--- a/arch/loongarch/kvm/mmu.c
++++ b/arch/loongarch/kvm/mmu.c
+@@ -679,19 +679,19 @@ static int host_pfn_mapping_level(struct
+ * value) and then p*d_offset() walks into the target huge page instead
+ * of the old page table (sees the new value).
+ */
+- pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
++ pgd = pgdp_get(pgd_offset(kvm->mm, hva));
+ if (pgd_none(pgd))
+ goto out;
+
+- p4d = READ_ONCE(*p4d_offset(&pgd, hva));
++ p4d = p4dp_get(p4d_offset(&pgd, hva));
+ if (p4d_none(p4d) || !p4d_present(p4d))
+ goto out;
+
+- pud = READ_ONCE(*pud_offset(&p4d, hva));
++ pud = pudp_get(pud_offset(&p4d, hva));
+ if (pud_none(pud) || !pud_present(pud))
+ goto out;
+
+- pmd = READ_ONCE(*pmd_offset(&pud, hva));
++ pmd = pmdp_get(pmd_offset(&pud, hva));
+ if (pmd_none(pmd) || !pmd_present(pmd))
+ goto out;
+
+--- a/arch/loongarch/mm/hugetlbpage.c
++++ b/arch/loongarch/mm/hugetlbpage.c
+@@ -39,11 +39,11 @@ pte_t *huge_pte_offset(struct mm_struct
+ pmd_t *pmd = NULL;
+
+ pgd = pgd_offset(mm, addr);
+- if (pgd_present(*pgd)) {
++ if (pgd_present(pgdp_get(pgd))) {
+ p4d = p4d_offset(pgd, addr);
+- if (p4d_present(*p4d)) {
++ if (p4d_present(p4dp_get(p4d))) {
+ pud = pud_offset(p4d, addr);
+- if (pud_present(*pud))
++ if (pud_present(pudp_get(pud)))
+ pmd = pmd_offset(pud, addr);
+ }
+ }
+--- a/arch/loongarch/mm/init.c
++++ b/arch/loongarch/mm/init.c
+@@ -140,7 +140,7 @@ void __meminit vmemmap_set_pmd(pmd_t *pm
+ int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
+ unsigned long addr, unsigned long next)
+ {
+- int huge = pmd_val(*pmd) & _PAGE_HUGE;
++ int huge = pmd_val(pmdp_get(pmd)) & _PAGE_HUGE;
+
+ if (huge)
+ vmemmap_verify((pte_t *)pmd, node, addr, next);
+@@ -172,7 +172,7 @@ pte_t * __init populate_kernel_pte(unsig
+ pud_t *pud;
+ pmd_t *pmd;
+
+- if (p4d_none(*p4d)) {
++ if (p4d_none(p4dp_get(p4d))) {
+ pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+ if (!pud)
+ panic("%s: Failed to allocate memory\n", __func__);
+@@ -183,7 +183,7 @@ pte_t * __init populate_kernel_pte(unsig
+ }
+
+ pud = pud_offset(p4d, addr);
+- if (pud_none(*pud)) {
++ if (pud_none(pudp_get(pud))) {
+ pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+ if (!pmd)
+ panic("%s: Failed to allocate memory\n", __func__);
+@@ -194,7 +194,7 @@ pte_t * __init populate_kernel_pte(unsig
+ }
+
+ pmd = pmd_offset(pud, addr);
+- if (!pmd_present(*pmd)) {
++ if (!pmd_present(pmdp_get(pmd))) {
+ pte_t *pte;
+
+ pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+@@ -215,7 +215,7 @@ void __init __set_fixmap(enum fixed_addr
+ BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
+
+ ptep = populate_kernel_pte(addr);
+- if (!pte_none(*ptep)) {
++ if (!pte_none(ptep_get(ptep))) {
+ pte_ERROR(*ptep);
+ return;
+ }
+--- a/arch/loongarch/mm/kasan_init.c
++++ b/arch/loongarch/mm/kasan_init.c
+@@ -105,7 +105,7 @@ static phys_addr_t __init kasan_alloc_ze
+
+ static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, bool early)
+ {
+- if (__pmd_none(early, READ_ONCE(*pmdp))) {
++ if (__pmd_none(early, pmdp_get(pmdp))) {
+ phys_addr_t pte_phys = early ?
+ __pa_symbol(kasan_early_shadow_pte) : kasan_alloc_zeroed_page(node);
+ if (!early)
+@@ -118,7 +118,7 @@ static pte_t *__init kasan_pte_offset(pm
+
+ static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, bool early)
+ {
+- if (__pud_none(early, READ_ONCE(*pudp))) {
++ if (__pud_none(early, pudp_get(pudp))) {
+ phys_addr_t pmd_phys = early ?
+ __pa_symbol(kasan_early_shadow_pmd) : kasan_alloc_zeroed_page(node);
+ if (!early)
+@@ -131,7 +131,7 @@ static pmd_t *__init kasan_pmd_offset(pu
+
+ static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, bool early)
+ {
+- if (__p4d_none(early, READ_ONCE(*p4dp))) {
++ if (__p4d_none(early, p4dp_get(p4dp))) {
+ phys_addr_t pud_phys = early ?
+ __pa_symbol(kasan_early_shadow_pud) : kasan_alloc_zeroed_page(node);
+ if (!early)
+@@ -154,7 +154,7 @@ static void __init kasan_pte_populate(pm
+ : kasan_alloc_zeroed_page(node);
+ next = addr + PAGE_SIZE;
+ set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
+- } while (ptep++, addr = next, addr != end && __pte_none(early, READ_ONCE(*ptep)));
++ } while (ptep++, addr = next, addr != end && __pte_none(early, ptep_get(ptep)));
+ }
+
+ static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
+@@ -166,7 +166,7 @@ static void __init kasan_pmd_populate(pu
+ do {
+ next = pmd_addr_end(addr, end);
+ kasan_pte_populate(pmdp, addr, next, node, early);
+- } while (pmdp++, addr = next, addr != end && __pmd_none(early, READ_ONCE(*pmdp)));
++ } while (pmdp++, addr = next, addr != end && __pmd_none(early, pmdp_get(pmdp)));
+ }
+
+ static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
+--- a/arch/loongarch/mm/pgtable.c
++++ b/arch/loongarch/mm/pgtable.c
+@@ -128,7 +128,7 @@ pmd_t mk_pmd(struct page *page, pgprot_t
+ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp, pmd_t pmd)
+ {
+- *pmdp = pmd;
++ WRITE_ONCE(*pmdp, pmd);
+ flush_tlb_all();
+ }
+
projects / thirdparty / kernel / stable-queue.git / commitdiff
