mm: drop fake head checks

With fake head pages eliminated in the previous commit, remove the
supporting infrastructure:

  - page_fixed_fake_head(): no longer needed to detect fake heads;
  - page_is_fake_head(): no longer needed;
  - page_count_writable(): no longer needed for RCU protection;
  - RCU read_lock in page_ref_add_unless(): no longer needed;

This substantially simplifies compound_head() and page_ref_add_unless(),
removing both branches and RCU overhead from these hot paths.

RCU was required to serialize allocation of hugetlb page against
get_page_unless_zero() and prevent writing to read-only fake head.  It is
redundant without fake heads.

See bd225530a4c7 ("mm/hugetlb_vmemmap: fix race with speculative PFN
walkers") for more details.

synchronize_rcu() in mm/hugetlb_vmemmap.c will be removed by a separate
patch.

Link: https://lkml.kernel.org/r/20260227194302.274384-14-kas@kernel.org
Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
Reviewed-by: Muchun Song <muchun.song@linux.dev>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Baoquan He <bhe@redhat.com>
Cc: Christoph Lameter <cl@gentwo.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Frank van der Linden <fvdl@google.com>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 43876b108f0a4aafb24af9d14674933b5815f08d..b8eef2181598d19361a60efd9052ca3e2421606d 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -221,102 +221,15 @@ static __always_inline bool compound_info_has_mask(void)
        return is_power_of_2(sizeof(struct page));
 }
 
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
 DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
 
-/*
- * Return the real head page struct iff the @page is a fake head page, otherwise
- * return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
- */
-static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
-{
-       /* Fake heads only exists if compound_info_has_mask() is true */
-       if (!compound_info_has_mask())
-               return page;
-
-       if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
-               return page;
-
-       /*
-        * Only addresses aligned with PAGE_SIZE of struct page may be fake head
-        * struct page. The alignment check aims to avoid access the fields (
-        * e.g. compound_info) of the @page[1]. It can avoid touch a (possibly)
-        * cold cacheline in some cases.
-        */
-       if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
-           test_bit(PG_head, &page->flags.f)) {
-               /*
-                * We can safely access the field of the @page[1] with PG_head
-                * because the @page is a compound page composed with at least
-                * two contiguous pages.
-                */
-               unsigned long info = READ_ONCE(page[1].compound_info);
-
-               /* See set_compound_head() */
-               if (likely(info & 1)) {
-                       unsigned long p = (unsigned long)page;
-
-                       return (const struct page *)(p & info);
-               }
-       }
-       return page;
-}
-
-static __always_inline bool page_count_writable(const struct page *page, int u)
-{
-       if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
-               return true;
-
-       /*
-        * The refcount check is ordered before the fake-head check to prevent
-        * the following race:
-        *   CPU 1 (HVO)                     CPU 2 (speculative PFN walker)
-        *
-        *   page_ref_freeze()
-        *   synchronize_rcu()
-        *                                   rcu_read_lock()
-        *                                   page_is_fake_head() is false
-        *   vmemmap_remap_pte()
-        *   XXX: struct page[] becomes r/o
-        *
-        *   page_ref_unfreeze()
-        *                                   page_ref_count() is not zero
-        *
-        *                                   atomic_add_unless(&page->_refcount)
-        *                                   XXX: try to modify r/o struct page[]
-        *
-        * The refcount check also prevents modification attempts to other (r/o)
-        * tail pages that are not fake heads.
-        */
-       if (atomic_read_acquire(&page->_refcount) == u)
-               return false;
-
-       return page_fixed_fake_head(page) == page;
-}
-#else
-static inline const struct page *page_fixed_fake_head(const struct page *page)
-{
-       return page;
-}
-
-static inline bool page_count_writable(const struct page *page, int u)
-{
-       return true;
-}
-#endif
-
-static __always_inline int page_is_fake_head(const struct page *page)
-{
-       return page_fixed_fake_head(page) != page;
-}
-
 static __always_inline unsigned long _compound_head(const struct page *page)
 {
        unsigned long info = READ_ONCE(page->compound_info);
 
        /* Bit 0 encodes PageTail() */
        if (!(info & 1))
-               return (unsigned long)page_fixed_fake_head(page);
+               return (unsigned long)page;
 
        /*
         * If compound_info_has_mask() is false, the rest of compound_info is
@@ -396,7 +309,7 @@ static __always_inline void clear_compound_head(struct page *page)
 
 static __always_inline int PageTail(const struct page *page)
 {
-       return READ_ONCE(page->compound_info) & 1 || page_is_fake_head(page);
+       return READ_ONCE(page->compound_info) & 1;
 }
 
 static __always_inline int PageCompound(const struct page *page)
@@ -928,7 +841,7 @@ static __always_inline bool folio_test_head(const struct folio *folio)
 static __always_inline int PageHead(const struct page *page)
 {
        PF_POISONED_CHECK(page);
-       return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page);
+       return test_bit(PG_head, &page->flags.f);
 }
 
 __SETPAGEFLAG(Head, head, PF_ANY)

diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h
index 544150d1d5fd7462423de4f93f36aefd6268a564..490d0ad6e56dc69278d7de12d2fbf54bee975bc2 100644 (file)
--- a/include/linux/page_ref.h
+++ b/include/linux/page_ref.h
@@ -230,13 +230,7 @@ static inline int folio_ref_dec_return(struct folio *folio)
 
 static inline bool page_ref_add_unless(struct page *page, int nr, int u)
 {
-       bool ret = false;
-
-       rcu_read_lock();
-       /* avoid writing to the vmemmap area being remapped */
-       if (page_count_writable(page, u))
-               ret = atomic_add_unless(&page->_refcount, nr, u);
-       rcu_read_unlock();
+       bool ret = atomic_add_unless(&page->_refcount, nr, u);
 
        if (page_ref_tracepoint_active(page_ref_mod_unless))
                __page_ref_mod_unless(page, nr, ret);