mm: implement for_each_valid_pfn() for CONFIG_SPARSEMEM

Implement for_each_valid_pfn() based on two helper functions.

The first_valid_pfn() function largely mirrors pfn_valid(), calling into a
pfn_section_first_valid() helper which is trivial for the !VMEMMAP case,
and in the VMEMMAP case will skip to the next subsection as needed.

Since next_valid_pfn() knows that its argument *is* a valid PFN, it
doesn't need to do any checking at all while iterating over the low bits
within a (sub)section mask; the whole (sub)section is either present or
not.

Note that the VMEMMAP version of pfn_section_first_valid() may return a
value *higher* than end_pfn when skipping to the next subsection, and
first_valid_pfn() happily returns that higher value.  This is fine.

[dwmw2@infradead.org: fix next_valid_pfn() for sparsemem]
Link: https://lkml.kernel.org/r/c15100fcf6781a60b852c4dbb43bdc98a678fcf0.camel@infradead.org
Link: https://lkml.kernel.org/r/20250423133821.789413-4-dwmw2@infradead.org
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Marc Rutland <mark.rutland@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Ruihan Li <lrh2000@pku.edu.cn>
Cc: Will Deacon <will@kernel.org>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 230a29c2ed1a9b1d83775f81f554d874fe097f7c..b19a98c20de8a47a0f1f626128442fd91c0f2f4a 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2075,11 +2075,37 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
 
        return usage ? test_bit(idx, usage->subsection_map) : 0;
 }
+
+static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn)
+{
+       struct mem_section_usage *usage = READ_ONCE(ms->usage);
+       int idx = subsection_map_index(*pfn);
+       unsigned long bit;
+
+       if (!usage)
+               return false;
+
+       if (test_bit(idx, usage->subsection_map))
+               return true;
+
+       /* Find the next subsection that exists */
+       bit = find_next_bit(usage->subsection_map, SUBSECTIONS_PER_SECTION, idx);
+       if (bit == SUBSECTIONS_PER_SECTION)
+               return false;
+
+       *pfn = (*pfn & PAGE_SECTION_MASK) + (bit * PAGES_PER_SUBSECTION);
+       return true;
+}
 #else
 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
 {
        return 1;
 }
+
+static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn)
+{
+       return true;
+}
 #endif
 
 void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
@@ -2128,6 +2154,58 @@ static inline int pfn_valid(unsigned long pfn)
 
        return ret;
 }
+
+/* Returns end_pfn or higher if no valid PFN remaining in range */
+static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end_pfn)
+{
+       unsigned long nr = pfn_to_section_nr(pfn);
+
+       rcu_read_lock_sched();
+
+       while (nr <= __highest_present_section_nr && pfn < end_pfn) {
+               struct mem_section *ms = __pfn_to_section(pfn);
+
+               if (valid_section(ms) &&
+                   (early_section(ms) || pfn_section_first_valid(ms, &pfn))) {
+                       rcu_read_unlock_sched();
+                       return pfn;
+               }
+
+               /* Nothing left in this section? Skip to next section */
+               nr++;
+               pfn = section_nr_to_pfn(nr);
+       }
+
+       rcu_read_unlock_sched();
+       return end_pfn;
+}
+
+static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_pfn)
+{
+       pfn++;
+
+       if (pfn >= end_pfn)
+               return end_pfn;
+
+       /*
+        * Either every PFN within the section (or subsection for VMEMMAP) is
+        * valid, or none of them are. So there's no point repeating the check
+        * for every PFN; only call first_valid_pfn() again when crossing a
+        * (sub)section boundary (i.e. !(pfn & ~PAGE_{SUB,}SECTION_MASK)).
+        */
+       if (pfn & ~(IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP) ?
+                  PAGE_SUBSECTION_MASK : PAGE_SECTION_MASK))
+               return pfn;
+
+       return first_valid_pfn(pfn, end_pfn);
+}
+
+
+#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn)                 \
+       for ((_pfn) = first_valid_pfn((_start_pfn), (_end_pfn));        \
+            (_pfn) < (_end_pfn);                                       \
+            (_pfn) = next_valid_pfn((_pfn), (_end_pfn)))
+
 #endif
 
 static inline int pfn_in_present_section(unsigned long pfn)