static struct kmem_cache *mm_slot_cache __ro_after_init;
+#define KHUGEPAGED_MIN_MTHP_ORDER 2
+
struct collapse_control {
bool is_khugepaged;
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
+
+ /* Each bit represents a single occupied (!none/zero) page. */
+ DECLARE_BITMAP(mthp_present_ptes, MAX_PTRS_PER_PTE);
};
/**
return result;
}
+/* Return the highest naturally aligned order that fits at @offset within a PMD. */
+static unsigned int max_order_from_offset(unsigned int offset)
+{
+ if (offset == 0)
+ return HPAGE_PMD_ORDER;
+
+ return min_t(unsigned int, __ffs(offset), HPAGE_PMD_ORDER);
+}
+
+/*
+ * mthp_collapse() consumes the bitmap that is generated during
+ * collapse_scan_pmd() to determine what regions and mTHP orders fit best.
+ *
+ * Each bit in cc->mthp_present_ptes represents a single occupied (!none/zero)
+ * page. We start at the PMD order and check if it is eligible for collapse;
+ * if not, we check the left and right halves of the PTE page table we are
+ * examining at a lower order.
+ *
+ * For each of these, we determine how many PTE entries are occupied in the
+ * range of PTE entries we propose to collapse, then we compare this to a
+ * threshold number of PTE entries which would need to be occupied for a
+ * collapse to be permitted at that order (accounting for max_ptes_none).
+ *
+ * If a collapse is permitted, we attempt to collapse the PTE range into a
+ * mTHP.
+ */
+static enum scan_result mthp_collapse(struct mm_struct *mm,
+ unsigned long address, int referenced, int unmapped,
+ struct collapse_control *cc, unsigned long enabled_orders)
+{
+ unsigned int nr_occupied_ptes, nr_ptes, max_ptes_none;
+ enum scan_result last_result = SCAN_FAIL;
+ int collapsed = 0;
+ bool alloc_failed = false;
+ unsigned long collapse_address;
+ unsigned int offset = 0;
+ unsigned int order = HPAGE_PMD_ORDER;
+
+ while (offset < HPAGE_PMD_NR) {
+ nr_ptes = 1UL << order;
+
+ if (!test_bit(order, &enabled_orders))
+ goto next_order;
+
+ max_ptes_none = collapse_max_ptes_none(cc, NULL, order);
+ nr_occupied_ptes = bitmap_weight_from(cc->mthp_present_ptes, offset,
+ offset + nr_ptes);
+
+ if (nr_occupied_ptes >= nr_ptes - max_ptes_none) {
+ enum scan_result ret;
+
+ collapse_address = address + offset * PAGE_SIZE;
+ ret = collapse_huge_page(mm, collapse_address, referenced,
+ unmapped, cc, order);
+ switch (ret) {
+ /* Cases where we continue to next collapse candidate */
+ case SCAN_SUCCEED:
+ collapsed += nr_ptes;
+ fallthrough;
+ case SCAN_PTE_MAPPED_HUGEPAGE:
+ goto next_offset;
+ /* Cases where lower orders might still succeed */
+ case SCAN_ALLOC_HUGE_PAGE_FAIL:
+ alloc_failed = true;
+ last_result = ret;
+ goto next_order;
+ /* Cases where no further collapse is possible */
+ case SCAN_PMD_MAPPED:
+ fallthrough;
+ default:
+ last_result = ret;
+ goto done;
+ }
+ }
+
+next_order:
+ /*
+ * Continue with the next smaller order if there is still
+ * any smaller order enabled. When at the smallest order
+ * we must always move to the next offset.
+ */
+ if (order > KHUGEPAGED_MIN_MTHP_ORDER &&
+ (enabled_orders & GENMASK(order - 1, 0))) {
+ order--;
+ continue;
+ }
+next_offset:
+ /*
+ * Advance past the region we just processed and determine the
+ * highest order we can attempt next. Since huge pages must be
+ * naturally aligned, the max order we can attempt next is
+ * limited by the alignment of the new offset.
+ * E.g. if we collapsed a order-2 mTHP at offset 0, offset
+ * becomes 4 and __ffs(4) == 2, so the next attempt starts at
+ * order 2.
+ */
+ offset += nr_ptes;
+ order = max_order_from_offset(offset);
+ }
+done:
+ if (collapsed)
+ return SCAN_SUCCEED;
+ if (alloc_failed)
+ return SCAN_ALLOC_HUGE_PAGE_FAIL;
+ return last_result;
+}
+
static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long start_addr,
bool *lock_dropped, struct collapse_control *cc)
{
- const unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER);
const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
+ unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
+ enum tva_type tva_flags = cc->is_khugepaged ? TVA_KHUGEPAGED : TVA_FORCED_COLLAPSE;
pmd_t *pmd;
- pte_t *pte, *_pte;
+ pte_t *pte, *_pte, pteval;
+ int i;
int none_or_zero = 0, shared = 0, referenced = 0;
enum scan_result result = SCAN_FAIL;
struct page *page = NULL;
struct folio *folio = NULL;
unsigned long addr;
+ unsigned long enabled_orders;
spinlock_t *ptl;
int node = NUMA_NO_NODE, unmapped = 0;
goto out;
}
+ bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE);
memset(cc->node_load, 0, sizeof(cc->node_load));
nodes_clear(cc->alloc_nmask);
+
+ enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags);
+
+ /*
+ * If PMD is the only enabled order, enforce max_ptes_none, otherwise
+ * scan all pages to populate the bitmap for mTHP collapse.
+ */
+ if (enabled_orders != BIT(HPAGE_PMD_ORDER))
+ max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
+
pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
if (!pte) {
cc->progress++;
goto out;
}
- for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR;
- _pte++, addr += PAGE_SIZE) {
+ for (i = 0; i < HPAGE_PMD_NR; i++) {
+ _pte = pte + i;
+ addr = start_addr + i * PAGE_SIZE;
+ pteval = ptep_get(_pte);
+
cc->progress++;
- pte_t pteval = ptep_get(_pte);
if (pte_none_or_zero(pteval)) {
if (++none_or_zero > max_ptes_none) {
result = SCAN_EXCEED_NONE_PTE;
}
}
+ /* Set bit for occupied pages */
+ __set_bit(i, cc->mthp_present_ptes);
/*
* Record which node the original page is from and save this
* information to cc->node_load[].
if (result == SCAN_SUCCEED) {
/* collapse_huge_page expects the lock to be dropped before calling */
mmap_read_unlock(mm);
- result = collapse_huge_page(mm, start_addr, referenced,
- unmapped, cc, HPAGE_PMD_ORDER);
- /* collapse_huge_page will return with the mmap_lock released */
+ result = mthp_collapse(mm, start_addr, referenced,
+ unmapped, cc, enabled_orders);
+ /* mmap_lock was released above, set lock_dropped */
*lock_dropped = true;
}
out:
projects / thirdparty / kernel / linux.git / commitdiff
