mm: hugetlb: Add huge page size param to huge_ptep_get_and_clear()

commit 02410ac72ac3707936c07ede66e94360d0d65319 upstream.

In order to fix a bug, arm64 needs to be told the size of the huge page
for which the huge_pte is being cleared in huge_ptep_get_and_clear().
Provide for this by adding an `unsigned long sz` parameter to the
function. This follows the same pattern as huge_pte_clear() and
set_huge_pte_at().

This commit makes the required interface modifications to the core mm as
well as all arches that implement this function (arm64, loongarch, mips,
parisc, powerpc, riscv, s390, sparc). The actual arm64 bug will be fixed
in a separate commit.

Cc: stable@vger.kernel.org
Fixes: 66b3923a1a0f ("arm64: hugetlb: add support for PTE contiguous bit")
Acked-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Alexandre Ghiti <alexghiti@rivosinc.com> # riscv
Reviewed-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Acked-by: Alexander Gordeev <agordeev@linux.ibm.com> # s390
Link: https://lore.kernel.org/r/20250226120656.2400136-2-ryan.roberts@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h
index 293f880865e8d0a27b4251fe50fb0784240ebbcf..f0304273eb351906bc020b43823483b56ad3f77c 100644 (file)
--- a/arch/arm64/include/asm/hugetlb.h
+++ b/arch/arm64/include/asm/hugetlb.h
@@ -34,8 +34,8 @@ extern int huge_ptep_set_access_flags(struct vm_area_struct *vma,
                                      unsigned long addr, pte_t *ptep,
                                      pte_t pte, int dirty);
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
-extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                                    unsigned long addr, pte_t *ptep);
+extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
+                                    pte_t *ptep, unsigned long sz);
 #define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT
 extern void huge_ptep_set_wrprotect(struct mm_struct *mm,
                                    unsigned long addr, pte_t *ptep);

diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 0a6956bbfb3269b1fa5df49400b4d08543f5f517..2e6b1f00e2e8111152a9f2682cb381f23c029e9c 100644 (file)
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -385,8 +385,8 @@ void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
                __pte_clear(mm, addr, ptep);
 }
 
-pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                             unsigned long addr, pte_t *ptep)
+pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
+                             pte_t *ptep, unsigned long sz)
 {
        int ncontig;
        size_t pgsize;
@@ -538,6 +538,8 @@ bool __init arch_hugetlb_valid_size(unsigned long size)
 
 pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
 {
+       unsigned long psize = huge_page_size(hstate_vma(vma));
+
        if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
                /*
                 * Break-before-make (BBM) is required for all user space mappings
@@ -547,7 +549,7 @@ pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr
                if (pte_user_exec(__ptep_get(ptep)))
                        return huge_ptep_clear_flush(vma, addr, ptep);
        }
-       return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+       return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
 }
 
 void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,

diff --git a/arch/loongarch/include/asm/hugetlb.h b/arch/loongarch/include/asm/hugetlb.h
index 376c0708e2979bf5f4988b7c65b3e5102c4c78b7..6302e60fbaee1ac9d4b6a51cdc7d7039556123b8 100644 (file)
--- a/arch/loongarch/include/asm/hugetlb.h
+++ b/arch/loongarch/include/asm/hugetlb.h
@@ -41,7 +41,8 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                                           unsigned long addr, pte_t *ptep)
+                                           unsigned long addr, pte_t *ptep,
+                                           unsigned long sz)
 {
        pte_t clear;
        pte_t pte = ptep_get(ptep);
@@ -56,8 +57,9 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
                                          unsigned long addr, pte_t *ptep)
 {
        pte_t pte;
+       unsigned long sz = huge_page_size(hstate_vma(vma));
 
-       pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+       pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
        flush_tlb_page(vma, addr);
        return pte;
 }

diff --git a/arch/mips/include/asm/hugetlb.h b/arch/mips/include/asm/hugetlb.h
index fd69c88085542e370b2a09df004a782fec48729d..00ee3c0366305c09981f0d867c4b1dfe6f409d5a 100644 (file)
--- a/arch/mips/include/asm/hugetlb.h
+++ b/arch/mips/include/asm/hugetlb.h
@@ -32,7 +32,8 @@ static inline int prepare_hugepage_range(struct file *file,
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                                           unsigned long addr, pte_t *ptep)
+                                           unsigned long addr, pte_t *ptep,
+                                           unsigned long sz)
 {
        pte_t clear;
        pte_t pte = *ptep;
@@ -47,13 +48,14 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
                                          unsigned long addr, pte_t *ptep)
 {
        pte_t pte;
+       unsigned long sz = huge_page_size(hstate_vma(vma));
 
        /*
         * clear the huge pte entry firstly, so that the other smp threads will
         * not get old pte entry after finishing flush_tlb_page and before
         * setting new huge pte entry
         */
-       pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+       pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
        flush_tlb_page(vma, addr);
        return pte;
 }

diff --git a/arch/parisc/include/asm/hugetlb.h b/arch/parisc/include/asm/hugetlb.h
index 72daacc472a0a305772d7473aa8601a5316fbbe4..f7a91411dcc955c564844ab87d0ddc76fd728781 100644 (file)
--- a/arch/parisc/include/asm/hugetlb.h
+++ b/arch/parisc/include/asm/hugetlb.h
@@ -10,7 +10,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
-                             pte_t *ptep);
+                             pte_t *ptep, unsigned long sz);
 
 /*
  * If the arch doesn't supply something else, assume that hugepage

diff --git a/arch/parisc/mm/hugetlbpage.c b/arch/parisc/mm/hugetlbpage.c
index aa664f7ddb63986cdc0ad61ad7b4a0457f919e41..cec2b9a581dd3e5e83c7f3bba2b15b6e8dcc9d6c 100644 (file)
--- a/arch/parisc/mm/hugetlbpage.c
+++ b/arch/parisc/mm/hugetlbpage.c
@@ -147,7 +147,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 
 
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
-                             pte_t *ptep)
+                             pte_t *ptep, unsigned long sz)
 {
        pte_t entry;
 

diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index dad2e7980f245bfe7240a968409acdf2315e48e2..86326587e58de803ef37b52ef519f50a2541a4fb 100644 (file)
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -45,7 +45,8 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                                           unsigned long addr, pte_t *ptep)
+                                           unsigned long addr, pte_t *ptep,
+                                           unsigned long sz)
 {
        return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1));
 }
@@ -55,8 +56,9 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
                                          unsigned long addr, pte_t *ptep)
 {
        pte_t pte;
+       unsigned long sz = huge_page_size(hstate_vma(vma));
 
-       pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+       pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
        flush_hugetlb_page(vma, addr);
        return pte;
 }

diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h
index faf3624d80577c68cef1c3e5a6fd3b4ab615088b..4461264977684b1f306a3435a5533b64dbd366a1 100644 (file)
--- a/arch/riscv/include/asm/hugetlb.h
+++ b/arch/riscv/include/asm/hugetlb.h
@@ -28,7 +28,8 @@ void set_huge_pte_at(struct mm_struct *mm,
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                             unsigned long addr, pte_t *ptep);
+                             unsigned long addr, pte_t *ptep,
+                             unsigned long sz);
 
 #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
 pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,

diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c
index 42314f0939220a0fce9ed77fb7ce113ac163b997..b4a78a4b35cff53194485bf584d98bbe33cc75e3 100644 (file)
--- a/arch/riscv/mm/hugetlbpage.c
+++ b/arch/riscv/mm/hugetlbpage.c
@@ -293,7 +293,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
                              unsigned long addr,
-                             pte_t *ptep)
+                             pte_t *ptep, unsigned long sz)
 {
        pte_t orig_pte = ptep_get(ptep);
        int pte_num;

diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index cf1b5d6fb1a6295076d148b1597dd31f0666aaf9..4731a51241ba861cc12419cb20dfb10461f54fc2 100644 (file)
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -20,8 +20,15 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
                     pte_t *ptep, pte_t pte);
 pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
-pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                             unsigned long addr, pte_t *ptep);
+pte_t __huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
+                               pte_t *ptep);
+
+static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
+                                           unsigned long addr, pte_t *ptep,
+                                           unsigned long sz)
+{
+       return __huge_ptep_get_and_clear(mm, addr, ptep);
+}
 
 /*
  * If the arch doesn't supply something else, assume that hugepage
@@ -57,7 +64,7 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
                                          unsigned long address, pte_t *ptep)
 {
-       return huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
+       return __huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
 }
 
 static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
@@ -66,7 +73,7 @@ static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 {
        int changed = !pte_same(huge_ptep_get(vma->vm_mm, addr, ptep), pte);
        if (changed) {
-               huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+               __huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
                __set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
        }
        return changed;
@@ -75,7 +82,7 @@ static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
                                           unsigned long addr, pte_t *ptep)
 {
-       pte_t pte = huge_ptep_get_and_clear(mm, addr, ptep);
+       pte_t pte = __huge_ptep_get_and_clear(mm, addr, ptep);
        __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte));
 }
 

diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index ded0eff58a192a60d460a8e6eda6e9d30a94667d..9c1ba8c0cac61a10d1c3b3c91c4b37ab1ee33f17 100644 (file)
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -174,8 +174,8 @@ pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
        return __rste_to_pte(pte_val(*ptep));
 }
 
-pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-                             unsigned long addr, pte_t *ptep)
+pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
+                               unsigned long addr, pte_t *ptep)
 {
        pte_t pte = huge_ptep_get(mm, addr, ptep);
        pmd_t *pmdp = (pmd_t *) ptep;

diff --git a/arch/sparc/include/asm/hugetlb.h b/arch/sparc/include/asm/hugetlb.h
index c714ca6a05aa04b154d0b64f52cb3f7d4c20452b..e7a9cdd498dca6bf035b11e94e4951cc3f31d31b 100644 (file)
--- a/arch/sparc/include/asm/hugetlb.h
+++ b/arch/sparc/include/asm/hugetlb.h
@@ -20,7 +20,7 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
-                             pte_t *ptep);
+                             pte_t *ptep, unsigned long sz);
 
 #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,

diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index cc91ca7a1e182cf6f6de2c96be13c0e7e248bed1..c276d70a7479951994edb6e8aec384f9cfea1470 100644 (file)
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -368,7 +368,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 }
 
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
-                             pte_t *ptep)
+                             pte_t *ptep, unsigned long sz)
 {
        unsigned int i, nptes, orig_shift, shift;
        unsigned long size;

diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h
index 594d5905f6151281d17edd1e7cdd5b13deddc74c..215bf9f317cbfdef5c5367b2eadd1d1ed289e50c 100644 (file)
--- a/include/asm-generic/hugetlb.h
+++ b/include/asm-generic/hugetlb.h
@@ -84,7 +84,7 @@ static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 
 #ifndef __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-               unsigned long addr, pte_t *ptep)
+               unsigned long addr, pte_t *ptep, unsigned long sz)
 {
        return ptep_get_and_clear(mm, addr, ptep);
 }

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index e4697539b665a2639ecd72a34f7f8b56e4c77dcf..25a7b13574c28b8f9b8932aed56d164028f65fe1 100644 (file)
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -1009,7 +1009,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
 static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
                                                unsigned long addr, pte_t *ptep)
 {
-       return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+       unsigned long psize = huge_page_size(hstate_vma(vma));
+
+       return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
 }
 #endif
 

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index bdee6d3ab0e7e3c38331bf70bf596d28764999b3..1e9aa6de4e21ea743962a635e49af8176071b9e2 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5395,7 +5395,7 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
        if (src_ptl != dst_ptl)
                spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
 
-       pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
+       pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);
 
        if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
                huge_pte_clear(mm, new_addr, dst_pte, sz);
@@ -5570,7 +5570,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
                        set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
                }
 
-               pte = huge_ptep_get_and_clear(mm, address, ptep);
+               pte = huge_ptep_get_and_clear(mm, address, ptep, sz);
                tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
                if (huge_pte_dirty(pte))
                        set_page_dirty(page);