s390/mm: Complete ptep_get() conversion

Finalize commit c33c794828f2 ("mm: ptep_get() conversion") and
replace direct page table entry dereferencing with the proper
accessors (ptep_get(), pmdp_get(), etc.).

Override the default getter implementations even though they are
currently identical: pud_clear(), p4d_clear(), and pgd_clear()
require corresponding architecture-specific getters, but these
are not yet defined. This avoids a dependency loop.

Acked-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>

diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c
index 7d6cc4c85af05727d303a387268e52990dcd8a6a..ff6d58a476baef3e8d242ae64e86a12bf9415d80 100644 (file)
--- a/arch/s390/boot/vmem.c
+++ b/arch/s390/boot/vmem.c
@@ -338,7 +338,7 @@ static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long e
 
        pte = pte_offset_kernel(pmd, addr);
        for (; addr < end; addr += PAGE_SIZE, pte++) {
-               if (pte_none(*pte)) {
+               if (pte_none(ptep_get(pte))) {
                        if (kasan_pte_populate_zero_shadow(pte, mode))
                                continue;
                        entry = __pte(resolve_pa_may_alloc(addr, PAGE_SIZE, mode));
@@ -355,26 +355,27 @@ static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long e
                                 enum populate_mode mode)
 {
        unsigned long pa, next, pages = 0;
-       pmd_t *pmd, entry;
+       pmd_t *pmd, entry, large_entry;
        pte_t *pte;
 
        pmd = pmd_offset(pud, addr);
        for (; addr < end; addr = next, pmd++) {
                next = pmd_addr_end(addr, end);
-               if (pmd_none(*pmd)) {
+               entry = pmdp_get(pmd);
+               if (pmd_none(entry)) {
                        if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))
                                continue;
                        pa = try_get_large_pmd_pa(pmd, addr, next, mode);
                        if (pa != INVALID_PHYS_ADDR) {
-                               entry = __pmd(pa);
-                               entry = set_pmd_bit(entry, SEGMENT_KERNEL);
-                               set_pmd(pmd, entry);
+                               large_entry = __pmd(pa);
+                               large_entry = set_pmd_bit(large_entry, SEGMENT_KERNEL);
+                               set_pmd(pmd, large_entry);
                                pages++;
                                continue;
                        }
                        pte = boot_pte_alloc();
                        pmd_populate(&init_mm, pmd, pte);
-               } else if (pmd_leaf(*pmd)) {
+               } else if (pmd_leaf(entry)) {
                        continue;
                }
                pgtable_pte_populate(pmd, addr, next, mode);
@@ -387,26 +388,27 @@ static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long e
                                 enum populate_mode mode)
 {
        unsigned long pa, next, pages = 0;
-       pud_t *pud, entry;
+       pud_t *pud, entry, large_entry;
        pmd_t *pmd;
 
        pud = pud_offset(p4d, addr);
        for (; addr < end; addr = next, pud++) {
                next = pud_addr_end(addr, end);
-               if (pud_none(*pud)) {
+               entry = pudp_get(pud);
+               if (pud_none(entry)) {
                        if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))
                                continue;
                        pa = try_get_large_pud_pa(pud, addr, next, mode);
                        if (pa != INVALID_PHYS_ADDR) {
-                               entry = __pud(pa);
-                               entry = set_pud_bit(entry, REGION3_KERNEL);
-                               set_pud(pud, entry);
+                               large_entry = __pud(pa);
+                               large_entry = set_pud_bit(large_entry, REGION3_KERNEL);
+                               set_pud(pud, large_entry);
                                pages++;
                                continue;
                        }
                        pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY);
                        pud_populate(&init_mm, pud, pmd);
-               } else if (pud_leaf(*pud)) {
+               } else if (pud_leaf(entry)) {
                        continue;
                }
                pgtable_pmd_populate(pud, addr, next, mode);
@@ -425,7 +427,7 @@ static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long e
        p4d = p4d_offset(pgd, addr);
        for (; addr < end; addr = next, p4d++) {
                next = p4d_addr_end(addr, end);
-               if (p4d_none(*p4d)) {
+               if (p4d_none(p4dp_get(p4d))) {
                        if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))
                                continue;
                        pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY);
@@ -451,7 +453,7 @@ static void pgtable_populate(unsigned long addr, unsigned long end, enum populat
        pgd = pgd_offset(&init_mm, addr);
        for (; addr < end; addr = next, pgd++) {
                next = pgd_addr_end(addr, end);
-               if (pgd_none(*pgd)) {
+               if (pgd_none(pgdp_get(pgd))) {
                        if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))
                                continue;
                        p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY);

diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index 6983e52eaf81943da11daf456dea2f420f577d3d..e33a5b587ee4e6b70c6457f847dfdbe2a49349ea 100644 (file)
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -41,7 +41,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
                                  pte_t *ptep, unsigned long sz)
 {
-       if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
+       if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
                set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY));
        else
                set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY));

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 3197b8b372a2a8729ef737f2972380eeca24ebd4..f9a8a92fa160d4930a1b3b0587254c01f886ada0 100644 (file)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -983,27 +983,34 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
        WRITE_ONCE(*ptep, pte);
 }
 
-static inline void pgd_clear(pgd_t *pgd)
+#define ptep_get ptep_get
+static inline pte_t ptep_get(pte_t *ptep)
 {
-       if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
-               set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY));
+       return READ_ONCE(*ptep);
 }
 
-static inline void p4d_clear(p4d_t *p4d)
+#define pmdp_get pmdp_get
+static inline pmd_t pmdp_get(pmd_t *pmdp)
 {
-       if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
-               set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY));
+       return READ_ONCE(*pmdp);
 }
 
-static inline void pud_clear(pud_t *pud)
+#define pudp_get pudp_get
+static inline pud_t pudp_get(pud_t *pudp)
 {
-       if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
-               set_pud(pud, __pud(_REGION3_ENTRY_EMPTY));
+       return READ_ONCE(*pudp);
 }
 
-static inline void pmd_clear(pmd_t *pmdp)
+#define p4dp_get p4dp_get
+static inline p4d_t p4dp_get(p4d_t *p4dp)
 {
-       set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+       return READ_ONCE(*p4dp);
+}
+
+#define pgdp_get pgdp_get
+static inline pgd_t pgdp_get(pgd_t *pgdp)
+{
+       return READ_ONCE(*pgdp);
 }
 
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
@@ -1011,6 +1018,29 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt
        set_pte(ptep, __pte(_PAGE_INVALID));
 }
 
+static inline void pmd_clear(pmd_t *pmdp)
+{
+       set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+}
+
+static inline void pud_clear(pud_t *pud)
+{
+       if ((pud_val(pudp_get(pud)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
+               set_pud(pud, __pud(_REGION3_ENTRY_EMPTY));
+}
+
+static inline void p4d_clear(p4d_t *p4d)
+{
+       if ((p4d_val(p4dp_get(p4d)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
+               set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY));
+}
+
+static inline void pgd_clear(pgd_t *pgd)
+{
+       if ((pgd_val(pgdp_get(pgd)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
+               set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY));
+}
+
 /*
  * The following pte modification functions only work if
  * pte_present() is true. Undefined behaviour if not..
@@ -1169,7 +1199,7 @@ pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
 static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma,
                unsigned long addr, pte_t *ptep)
 {
-       pte_t pte = *ptep;
+       pte_t pte = ptep_get(ptep);
 
        pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
        return pte_young(pte);
@@ -1230,7 +1260,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
        pte_t res;
 
        if (full) {
-               res = *ptep;
+               res = ptep_get(ptep);
                set_pte(ptep, __pte(_PAGE_INVALID));
        } else {
                res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
@@ -1259,7 +1289,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 static inline void ptep_set_wrprotect(struct mm_struct *mm,
                                      unsigned long addr, pte_t *ptep)
 {
-       pte_t pte = *ptep;
+       pte_t pte = ptep_get(ptep);
 
        if (pte_write(pte))
                ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
@@ -1295,7 +1325,7 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
         * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead.
         * A local RDP can be used to do the flush.
         */
-       if (cpu_has_rdp() && !(pte_val(*ptep) & _PAGE_PROTECT))
+       if (cpu_has_rdp() && !(pte_val(ptep_get(ptep)) & _PAGE_PROTECT))
                __ptep_rdp(address, ptep, 1);
 }
 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault

diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index 302ef5781b6561eae982d1b353f5169ba55ce0d2..db35d8fe8609089b547e9614500cec800e01d217 100644 (file)
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -143,7 +143,7 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
        rste = __pte_to_rste(pte);
 
        /* Set correct table type for 2G hugepages */
-       if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
+       if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
                if (likely(pte_present(pte)))
                        rste |= _REGION3_ENTRY_LARGE;
                rste |= _REGION_ENTRY_TYPE_R3;
@@ -161,7 +161,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 
 pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
-       return __rste_to_pte(pte_val(*ptep));
+       return __rste_to_pte(pte_val(ptep_get(ptep)));
 }
 
 pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
@@ -171,7 +171,7 @@ pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
        pmd_t *pmdp = (pmd_t *) ptep;
        pud_t *pudp = (pud_t *) ptep;
 
-       if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
+       if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
                pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
        else
                pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
@@ -209,13 +209,13 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
        pmd_t *pmdp = NULL;
 
        pgdp = pgd_offset(mm, addr);
-       if (pgd_present(*pgdp)) {
+       if (pgd_present(pgdp_get(pgdp))) {
                p4dp = p4d_offset(pgdp, addr);
-               if (p4d_present(*p4dp)) {
+               if (p4d_present(p4dp_get(p4dp))) {
                        pudp = pud_offset(p4dp, addr);
                        if (sz == PUD_SIZE)
                                return (pte_t *)pudp;
-                       if (pud_present(*pudp))
+                       if (pud_present(pudp_get(pudp)))
                                pmdp = pmd_offset(pudp, addr);
                }
        }

diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index bb29c38ae6241d1c634e5c81920d0826b4b5b275..e6f788696dd1a98a18e2f214e327207de85bd1a2 100644 (file)
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -85,7 +85,7 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end,
                return 0;
        ptep = pte_offset_kernel(pmdp, addr);
        do {
-               new = *ptep;
+               new = ptep_get(ptep);
                if (pte_none(new))
                        return -EINVAL;
                if (flags & SET_MEMORY_RO)
@@ -114,15 +114,16 @@ static int split_pmd_page(pmd_t *pmdp, unsigned long addr)
 {
        unsigned long pte_addr, prot;
        pte_t *pt_dir, *ptep;
-       pmd_t new;
+       pmd_t new, pmd;
        int i, ro, nx;
 
        pt_dir = vmem_pte_alloc();
        if (!pt_dir)
                return -ENOMEM;
-       pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT;
-       ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT);
-       nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC);
+       pmd = pmdp_get(pmdp);
+       pte_addr = pmd_pfn(pmd) << PAGE_SHIFT;
+       ro = !!(pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT);
+       nx = !!(pmd_val(pmd) & _SEGMENT_ENTRY_NOEXEC);
        prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
        if (!nx)
                prot &= ~_PAGE_NOEXEC;
@@ -142,7 +143,7 @@ static int split_pmd_page(pmd_t *pmdp, unsigned long addr)
 static void modify_pmd_page(pmd_t *pmdp, unsigned long addr,
                            unsigned long flags)
 {
-       pmd_t new = *pmdp;
+       pmd_t new = pmdp_get(pmdp);
 
        if (flags & SET_MEMORY_RO)
                new = pmd_wrprotect(new);
@@ -165,16 +166,17 @@ static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end,
                          unsigned long flags)
 {
        unsigned long next;
+       pmd_t *pmdp, pmd;
        int need_split;
-       pmd_t *pmdp;
        int rc = 0;
 
        pmdp = pmd_offset(pudp, addr);
        do {
-               if (pmd_none(*pmdp))
+               pmd = pmdp_get(pmdp);
+               if (pmd_none(pmd))
                        return -EINVAL;
                next = pmd_addr_end(addr, end);
-               if (pmd_leaf(*pmdp)) {
+               if (pmd_leaf(pmd)) {
                        need_split  = !!(flags & SET_MEMORY_4K);
                        need_split |= !!(addr & ~PMD_MASK);
                        need_split |= !!(addr + PMD_SIZE > next);
@@ -201,15 +203,16 @@ int split_pud_page(pud_t *pudp, unsigned long addr)
 {
        unsigned long pmd_addr, prot;
        pmd_t *pm_dir, *pmdp;
-       pud_t new;
+       pud_t new, pud;
        int i, ro, nx;
 
        pm_dir = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY);
        if (!pm_dir)
                return -ENOMEM;
-       pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT;
-       ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT);
-       nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC);
+       pud = pudp_get(pudp);
+       pmd_addr = pud_pfn(pud) << PAGE_SHIFT;
+       ro = !!(pud_val(pud) & _REGION_ENTRY_PROTECT);
+       nx = !!(pud_val(pud) & _REGION_ENTRY_NOEXEC);
        prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL);
        if (!nx)
                prot &= ~_SEGMENT_ENTRY_NOEXEC;
@@ -229,7 +232,7 @@ int split_pud_page(pud_t *pudp, unsigned long addr)
 static void modify_pud_page(pud_t *pudp, unsigned long addr,
                            unsigned long flags)
 {
-       pud_t new = *pudp;
+       pud_t new = pudp_get(pudp);
 
        if (flags & SET_MEMORY_RO)
                new = pud_wrprotect(new);
@@ -252,16 +255,17 @@ static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end,
                          unsigned long flags)
 {
        unsigned long next;
+       pud_t *pudp, pud;
        int need_split;
-       pud_t *pudp;
        int rc = 0;
 
        pudp = pud_offset(p4d, addr);
        do {
-               if (pud_none(*pudp))
+               pud = pudp_get(pudp);
+               if (pud_none(pud))
                        return -EINVAL;
                next = pud_addr_end(addr, end);
-               if (pud_leaf(*pudp)) {
+               if (pud_leaf(pud)) {
                        need_split  = !!(flags & SET_MEMORY_4K);
                        need_split |= !!(addr & ~PUD_MASK);
                        need_split |= !!(addr + PUD_SIZE > next);
@@ -291,7 +295,7 @@ static int walk_p4d_level(pgd_t *pgd, unsigned long addr, unsigned long end,
 
        p4dp = p4d_offset(pgd, addr);
        do {
-               if (p4d_none(*p4dp))
+               if (p4d_none(p4dp_get(p4dp)))
                        return -EINVAL;
                next = p4d_addr_end(addr, end);
                rc = walk_pud_level(p4dp, addr, next, flags);
@@ -313,7 +317,7 @@ static int change_page_attr(unsigned long addr, unsigned long end,
 
        pgdp = pgd_offset_k(addr);
        do {
-               if (pgd_none(*pgdp))
+               if (pgd_none(pgdp_get(pgdp)))
                        break;
                next = pgd_addr_end(addr, end);
                rc = walk_p4d_level(pgdp, addr, next, flags);
@@ -451,7 +455,8 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)
                nr = min(numpages - i, nr);
                if (enable) {
                        for (j = 0; j < nr; j++) {
-                               pte = clear_pte_bit(*ptep, __pgprot(_PAGE_INVALID));
+                               pte = ptep_get(ptep);
+                               pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
                                set_pte(ptep, pte);
                                address += PAGE_SIZE;
                                ptep++;

diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index eeadff45e0e102165be6decd1fe7074d6285a23a..803099f3db73d6a2eb9e29be1870312e7a16b51f 100644 (file)
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -171,18 +171,19 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr,
 {
        unsigned long prot, pages = 0;
        int ret = -ENOMEM;
-       pte_t *pte;
+       pte_t *pte, entry;
 
        prot = pgprot_val(PAGE_KERNEL);
        pte = pte_offset_kernel(pmd, addr);
        for (; addr < end; addr += PAGE_SIZE, pte++) {
+               entry = ptep_get(pte);
                if (!add) {
-                       if (pte_none(*pte))
+                       if (pte_none(entry))
                                continue;
                        if (!direct)
-                               vmem_free_pages((unsigned long)pfn_to_virt(pte_pfn(*pte)), get_order(PAGE_SIZE), altmap);
+                               vmem_free_pages((unsigned long)pfn_to_virt(pte_pfn(entry)), get_order(PAGE_SIZE), altmap);
                        pte_clear(&init_mm, addr, pte);
-               } else if (pte_none(*pte)) {
+               } else if (pte_none(entry)) {
                        if (!direct) {
                                void *new_page = vmemmap_alloc_block_buf(PAGE_SIZE, NUMA_NO_NODE, altmap);
 
@@ -212,10 +213,10 @@ static void try_free_pte_table(pmd_t *pmd, unsigned long start)
        /* We can safely assume this is fully in 1:1 mapping & vmemmap area */
        pte = pte_offset_kernel(pmd, start);
        for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
-               if (!pte_none(*pte))
+               if (!pte_none(ptep_get(pte)))
                        return;
        }
-       vmem_pte_free((unsigned long *) pmd_deref(*pmd));
+       vmem_pte_free((unsigned long *)pmd_deref(pmdp_get(pmd)));
        pmd_clear(pmd);
 }
 
@@ -226,6 +227,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr,
 {
        unsigned long next, prot, pages = 0;
        int ret = -ENOMEM;
+       pmd_t entry;
        pmd_t *pmd;
        pte_t *pte;
 
@@ -233,23 +235,24 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr,
        pmd = pmd_offset(pud, addr);
        for (; addr < end; addr = next, pmd++) {
                next = pmd_addr_end(addr, end);
+               entry = pmdp_get(pmd);
                if (!add) {
-                       if (pmd_none(*pmd))
+                       if (pmd_none(entry))
                                continue;
-                       if (pmd_leaf(*pmd)) {
+                       if (pmd_leaf(entry)) {
                                if (IS_ALIGNED(addr, PMD_SIZE) &&
                                    IS_ALIGNED(next, PMD_SIZE)) {
                                        if (!direct)
-                                               vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE), altmap);
+                                               vmem_free_pages(pmd_deref(entry), get_order(PMD_SIZE), altmap);
                                        pmd_clear(pmd);
                                        pages++;
                                } else if (!direct && vmemmap_unuse_sub_pmd(addr, next)) {
-                                       vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE), altmap);
+                                       vmem_free_pages(pmd_deref(entry), get_order(PMD_SIZE), altmap);
                                        pmd_clear(pmd);
                                }
                                continue;
                        }
-               } else if (pmd_none(*pmd)) {
+               } else if (pmd_none(entry)) {
                        if (IS_ALIGNED(addr, PMD_SIZE) &&
                            IS_ALIGNED(next, PMD_SIZE) &&
                            cpu_has_edat1() && direct &&
@@ -281,7 +284,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr,
                        if (!pte)
                                goto out;
                        pmd_populate(&init_mm, pmd, pte);
-               } else if (pmd_leaf(*pmd)) {
+               } else if (pmd_leaf(entry)) {
                        if (!direct)
                                vmemmap_use_sub_pmd(addr, next);
                        continue;
@@ -306,9 +309,9 @@ static void try_free_pmd_table(pud_t *pud, unsigned long start)
 
        pmd = pmd_offset(pud, start);
        for (i = 0; i < PTRS_PER_PMD; i++, pmd++)
-               if (!pmd_none(*pmd))
+               if (!pmd_none(pmdp_get(pmd)))
                        return;
-       vmem_free_pages(pud_deref(*pud), CRST_ALLOC_ORDER, NULL);
+       vmem_free_pages(pud_deref(pudp_get(pud)), CRST_ALLOC_ORDER, NULL);
        pud_clear(pud);
 }
 
@@ -317,21 +320,22 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end,
 {
        unsigned long next, prot, pages = 0;
        int ret = -ENOMEM;
-       pud_t *pud;
+       pud_t *pud, entry;
        pmd_t *pmd;
 
        prot = pgprot_val(REGION3_KERNEL);
        pud = pud_offset(p4d, addr);
        for (; addr < end; addr = next, pud++) {
                next = pud_addr_end(addr, end);
+               entry = pudp_get(pud);
                if (!add) {
-                       if (pud_none(*pud))
+                       if (pud_none(entry))
                                continue;
-                       if (pud_leaf(*pud)) {
+                       if (pud_leaf(entry)) {
                                if (IS_ALIGNED(addr, PUD_SIZE) &&
                                    IS_ALIGNED(next, PUD_SIZE)) {
                                        if (!direct)
-                                               vmem_free_pages(pud_deref(*pud), get_order(PUD_SIZE), altmap);
+                                               vmem_free_pages(pud_deref(entry), get_order(PUD_SIZE), altmap);
                                        pud_clear(pud);
                                        pages++;
                                        continue;
@@ -339,7 +343,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end,
                                        split_pud_page(pud, addr & PUD_MASK);
                                }
                        }
-               } else if (pud_none(*pud)) {
+               } else if (pud_none(entry)) {
                        if (IS_ALIGNED(addr, PUD_SIZE) &&
                            IS_ALIGNED(next, PUD_SIZE) &&
                            cpu_has_edat2() && direct &&
@@ -352,7 +356,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end,
                        if (!pmd)
                                goto out;
                        pud_populate(&init_mm, pud, pmd);
-               } else if (pud_leaf(*pud)) {
+               } else if (pud_leaf(entry)) {
                        continue;
                }
                ret = modify_pmd_table(pud, addr, next, add, direct, altmap);
@@ -375,10 +379,10 @@ static void try_free_pud_table(p4d_t *p4d, unsigned long start)
 
        pud = pud_offset(p4d, start);
        for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
-               if (!pud_none(*pud))
+               if (!pud_none(pudp_get(pud)))
                        return;
        }
-       vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER, NULL);
+       vmem_free_pages(p4d_deref(p4dp_get(p4d)), CRST_ALLOC_ORDER, NULL);
        p4d_clear(p4d);
 }
 
@@ -387,16 +391,17 @@ static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end,
 {
        unsigned long next;
        int ret = -ENOMEM;
-       p4d_t *p4d;
+       p4d_t *p4d, entry;
        pud_t *pud;
 
        p4d = p4d_offset(pgd, addr);
        for (; addr < end; addr = next, p4d++) {
                next = p4d_addr_end(addr, end);
+               entry = p4dp_get(p4d);
                if (!add) {
-                       if (p4d_none(*p4d))
+                       if (p4d_none(entry))
                                continue;
-               } else if (p4d_none(*p4d)) {
+               } else if (p4d_none(entry)) {
                        pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY);
                        if (!pud)
                                goto out;
@@ -420,10 +425,10 @@ static void try_free_p4d_table(pgd_t *pgd, unsigned long start)
 
        p4d = p4d_offset(pgd, start);
        for (i = 0; i < PTRS_PER_P4D; i++, p4d++) {
-               if (!p4d_none(*p4d))
+               if (!p4d_none(p4dp_get(p4d)))
                        return;
        }
-       vmem_free_pages(pgd_deref(*pgd), CRST_ALLOC_ORDER, NULL);
+       vmem_free_pages(pgd_deref(pgdp_get(pgd)), CRST_ALLOC_ORDER, NULL);
        pgd_clear(pgd);
 }
 
@@ -432,7 +437,7 @@ static int modify_pagetable(unsigned long start, unsigned long end, bool add,
 {
        unsigned long addr, next;
        int ret = -ENOMEM;
-       pgd_t *pgd;
+       pgd_t *pgd, entry;
        p4d_t *p4d;
 
        if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end)))
@@ -449,11 +454,12 @@ static int modify_pagetable(unsigned long start, unsigned long end, bool add,
        for (addr = start; addr < end; addr = next) {
                next = pgd_addr_end(addr, end);
                pgd = pgd_offset_k(addr);
+               entry = pgdp_get(pgd);
 
                if (!add) {
-                       if (pgd_none(*pgd))
+                       if (pgd_none(entry))
                                continue;
-               } else if (pgd_none(*pgd)) {
+               } else if (pgd_none(entry)) {
                        p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY);
                        if (!p4d)
                                goto out;
@@ -575,6 +581,8 @@ int vmem_add_mapping(unsigned long start, unsigned long size)
 pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
 {
        pte_t *ptep = NULL;
+       pud_t pud_entry;
+       pmd_t pmd_entry;
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
@@ -582,7 +590,7 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
        pte_t *pte;
 
        pgd = pgd_offset_k(addr);
-       if (pgd_none(*pgd)) {
+       if (pgd_none(pgdp_get(pgd))) {
                if (!alloc)
                        goto out;
                p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY);
@@ -591,7 +599,7 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
                pgd_populate(&init_mm, pgd, p4d);
        }
        p4d = p4d_offset(pgd, addr);
-       if (p4d_none(*p4d)) {
+       if (p4d_none(p4dp_get(p4d))) {
                if (!alloc)
                        goto out;
                pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY);
@@ -600,25 +608,27 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
                p4d_populate(&init_mm, p4d, pud);
        }
        pud = pud_offset(p4d, addr);
-       if (pud_none(*pud)) {
+       pud_entry = pudp_get(pud);
+       if (pud_none(pud_entry)) {
                if (!alloc)
                        goto out;
                pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY);
                if (!pmd)
                        goto out;
                pud_populate(&init_mm, pud, pmd);
-       } else if (WARN_ON_ONCE(pud_leaf(*pud))) {
+       } else if (WARN_ON_ONCE(pud_leaf(pud_entry))) {
                goto out;
        }
        pmd = pmd_offset(pud, addr);
-       if (pmd_none(*pmd)) {
+       pmd_entry = pmdp_get(pmd);
+       if (pmd_none(pmd_entry)) {
                if (!alloc)
                        goto out;
                pte = vmem_pte_alloc();
                if (!pte)
                        goto out;
                pmd_populate(&init_mm, pmd, pte);
-       } else if (WARN_ON_ONCE(pmd_leaf(*pmd))) {
+       } else if (WARN_ON_ONCE(pmd_leaf(pmd_entry))) {
                goto out;
        }
        ptep = pte_offset_kernel(pmd, addr);