mm/khugepaged: generalize alloc_charge_folio()

Pass order to alloc_charge_folio() and update mTHP statistics.

Link: https://lore.kernel.org/20260605161422.213817-3-npache@redhat.com
Signed-off-by: Dev Jain <dev.jain@arm.com>
Co-developed-by: Nico Pache <npache@redhat.com>
Signed-off-by: Nico Pache <npache@redhat.com>
Reviewed-by: Wei Yang <richard.weiyang@gmail.com>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Acked-by: Usama Arif <usama.arif@linux.dev>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Bagas Sanjaya <bagasdotme@gmail.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nanyong Sun <sunnanyong@huawei.com>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Peter Xu <peterx@redhat.com>
Cc: Rafael Aquini <raquini@redhat.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Shivank Garg <shivankg@amd.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Takashi Iwai (SUSE) <tiwai@suse.de>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Cc: Yang Shi <yang@os.amperecomputing.com>
Cc: Zach O'Keefe <zokeefe@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
index 76f4eb14e26287ab63c1123a1e3e2e7db673aac8..a74844e01f1e86e982f36da5994f52fa949db63a 100644 (file)
--- a/Documentation/admin-guide/mm/transhuge.rst
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -639,6 +639,14 @@ anon_fault_fallback_charge
        instead falls back to using huge pages with lower orders or
        small pages even though the allocation was successful.
 
+collapse_alloc
+       is incremented every time a huge page is successfully allocated for a
+       khugepaged collapse.
+
+collapse_alloc_failed
+       is incremented every time a huge page allocation fails during a
+       khugepaged collapse.
+
 zswpout
        is incremented every time a huge page is swapped out to zswap in one
        piece without splitting.

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index c0d223d0c556c8caab5f32608efa7a152e10a050..e225d1d4a3fef61c7ab9b43cb12e3087f7b3a923 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -128,6 +128,8 @@ enum mthp_stat_item {
        MTHP_STAT_ANON_FAULT_ALLOC,
        MTHP_STAT_ANON_FAULT_FALLBACK,
        MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE,
+       MTHP_STAT_COLLAPSE_ALLOC,
+       MTHP_STAT_COLLAPSE_ALLOC_FAILED,
        MTHP_STAT_ZSWPOUT,
        MTHP_STAT_SWPIN,
        MTHP_STAT_SWPIN_FALLBACK,

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index a5176653ba1f214c350ce16689f374f8ea5cb0d8..c8b851dd518afdc878f9c5af528a800778014ab9 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -702,6 +702,8 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC);
 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK);
 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
+DEFINE_MTHP_STAT_ATTR(collapse_alloc, MTHP_STAT_COLLAPSE_ALLOC);
+DEFINE_MTHP_STAT_ATTR(collapse_alloc_failed, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT);
 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN);
 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK);
@@ -767,6 +769,8 @@ static struct attribute *any_stats_attrs[] = {
 #endif
        &split_attr.attr,
        &split_failed_attr.attr,
+       &collapse_alloc_attr.attr,
+       &collapse_alloc_failed_attr.attr,
        NULL,
 };
 

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index cc1328cb62376345fe6c8240ff2019026f493bf9..454ca90e3696fc399686d4901b62ab878396394f 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1077,28 +1077,34 @@ out:
 }
 
 static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm,
-               struct collapse_control *cc)
+               struct collapse_control *cc, unsigned int order)
 {
        gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :
                     GFP_TRANSHUGE);
        int node = collapse_find_target_node(cc);
        struct folio *folio;
 
-       folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask);
+       folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask);
        if (!folio) {
                *foliop = NULL;
-               count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+               if (is_pmd_order(order))
+                       count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+               count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
                return SCAN_ALLOC_HUGE_PAGE_FAIL;
        }
 
-       count_vm_event(THP_COLLAPSE_ALLOC);
+       if (is_pmd_order(order))
+               count_vm_event(THP_COLLAPSE_ALLOC);
+       count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC);
+
        if (unlikely(mem_cgroup_charge(folio, mm, gfp))) {
                folio_put(folio);
                *foliop = NULL;
                return SCAN_CGROUP_CHARGE_FAIL;
        }
 
-       count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1);
+       if (is_pmd_order(order))
+               count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1);
 
        *foliop = folio;
        return SCAN_SUCCEED;
@@ -1127,7 +1133,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
         */
        mmap_read_unlock(mm);
 
-       result = alloc_charge_folio(&folio, mm, cc);
+       result = alloc_charge_folio(&folio, mm, cc, HPAGE_PMD_ORDER);
        if (result != SCAN_SUCCEED)
                goto out_nolock;
 
@@ -1913,7 +1919,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
        VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
        VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
 
-       result = alloc_charge_folio(&new_folio, mm, cc);
+       result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER);
        if (result != SCAN_SUCCEED)
                goto out;