[thirdparty/linux.git] / mm / page_vma_mapped.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/rmap.h>
#include <linux/hugetlb.h>
#include <linux/swap.h>
#include <linux/swapops.h>

#include "internal.h"

static inline bool not_found(struct page_vma_mapped_walk *pvmw)
{
	page_vma_mapped_walk_done(pvmw);
	return false;
}

static bool map_pte(struct page_vma_mapped_walk *pvmw, spinlock_t **ptlp)
{
	pte_t ptent;

	if (pvmw->flags & PVMW_SYNC) {
		/* Use the stricter lookup */
		pvmw->pte = pte_offset_map_lock(pvmw->vma->vm_mm, pvmw->pmd,
						pvmw->address, &pvmw->ptl);
		*ptlp = pvmw->ptl;
		return !!pvmw->pte;
	}

	/*
	 * It is important to return the ptl corresponding to pte,
	 * in case *pvmw->pmd changes underneath us; so we need to
	 * return it even when choosing not to lock, in case caller
	 * proceeds to loop over next ptes, and finds a match later.
	 * Though, in most cases, page lock already protects this.
	 */
	pvmw->pte = pte_offset_map_nolock(pvmw->vma->vm_mm, pvmw->pmd,
					  pvmw->address, ptlp);
	if (!pvmw->pte)
		return false;

	ptent = ptep_get(pvmw->pte);

	if (pvmw->flags & PVMW_MIGRATION) {
		if (!is_swap_pte(ptent))
			return false;
	} else if (is_swap_pte(ptent)) {
		swp_entry_t entry;
		/*
		 * Handle un-addressable ZONE_DEVICE memory.
		 *
		 * We get here when we are trying to unmap a private
		 * device page from the process address space. Such
		 * page is not CPU accessible and thus is mapped as
		 * a special swap entry, nonetheless it still does
		 * count as a valid regular mapping for the page
		 * (and is accounted as such in page maps count).
		 *
		 * So handle this special case as if it was a normal
		 * page mapping ie lock CPU page table and return true.
		 *
		 * For more details on device private memory see HMM
		 * (include/linux/hmm.h or mm/hmm.c).
		 */
		entry = pte_to_swp_entry(ptent);
		if (!is_device_private_entry(entry) &&
		    !is_device_exclusive_entry(entry))
			return false;
	} else if (!pte_present(ptent)) {
		return false;
	}
	pvmw->ptl = *ptlp;
	spin_lock(pvmw->ptl);
	return true;
}

/**
 * check_pte - check if [pvmw->pfn, @pvmw->pfn + @pvmw->nr_pages) is
 * mapped at the @pvmw->pte
 * @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range
 * for checking
 *
 * page_vma_mapped_walk() found a place where pfn range is *potentially*
 * mapped. check_pte() has to validate this.
 *
 * pvmw->pte may point to empty PTE, swap PTE or PTE pointing to
 * arbitrary page.
 *
 * If PVMW_MIGRATION flag is set, returns true if @pvmw->pte contains migration
 * entry that points to [pvmw->pfn, @pvmw->pfn + @pvmw->nr_pages)
 *
 * If PVMW_MIGRATION flag is not set, returns true if pvmw->pte points to
 * [pvmw->pfn, @pvmw->pfn + @pvmw->nr_pages)
 *
 * Otherwise, return false.
 *
 */
static bool check_pte(struct page_vma_mapped_walk *pvmw)
{
	unsigned long pfn;
	pte_t ptent = ptep_get(pvmw->pte);

	if (pvmw->flags & PVMW_MIGRATION) {
		swp_entry_t entry;
		if (!is_swap_pte(ptent))
			return false;
		entry = pte_to_swp_entry(ptent);

		if (!is_migration_entry(entry) &&
		    !is_device_exclusive_entry(entry))
			return false;

		pfn = swp_offset_pfn(entry);
	} else if (is_swap_pte(ptent)) {
		swp_entry_t entry;

		/* Handle un-addressable ZONE_DEVICE memory */
		entry = pte_to_swp_entry(ptent);
		if (!is_device_private_entry(entry) &&
		    !is_device_exclusive_entry(entry))
			return false;

		pfn = swp_offset_pfn(entry);
	} else {
		if (!pte_present(ptent))
			return false;

		pfn = pte_pfn(ptent);
	}

	return (pfn - pvmw->pfn) < pvmw->nr_pages;
}

/* Returns true if the two ranges overlap.  Careful to not overflow. */
static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
{
	if ((pfn + HPAGE_PMD_NR - 1) < pvmw->pfn)
		return false;
	if (pfn > pvmw->pfn + pvmw->nr_pages - 1)
		return false;
	return true;
}

static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
{
	pvmw->address = (pvmw->address + size) & ~(size - 1);
	if (!pvmw->address)
		pvmw->address = ULONG_MAX;
}

/**
 * page_vma_mapped_walk - check if @pvmw->pfn is mapped in @pvmw->vma at
 * @pvmw->address
 * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
 * must be set. pmd, pte and ptl must be NULL.
 *
 * Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point
 * to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is
 * adjusted if needed (for PTE-mapped THPs).
 *
 * If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page
 * (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in
 * a loop to find all PTEs that map the THP.
 *
 * For HugeTLB pages, @pvmw->pte is set to the relevant page table entry
 * regardless of which page table level the page is mapped at. @pvmw->pmd is
 * NULL.
 *
 * Returns false if there are no more page table entries for the page in
 * the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped.
 *
 * If you need to stop the walk before page_vma_mapped_walk() returned false,
 * use page_vma_mapped_walk_done(). It will do the housekeeping.
 */
bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
{
	struct vm_area_struct *vma = pvmw->vma;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long end;
	spinlock_t *ptl;
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t pmde;

	/* The only possible pmd mapping has been handled on last iteration */
	if (pvmw->pmd && !pvmw->pte)
		return not_found(pvmw);

	if (unlikely(is_vm_hugetlb_page(vma))) {
		struct hstate *hstate = hstate_vma(vma);
		unsigned long size = huge_page_size(hstate);
		/* The only possible mapping was handled on last iteration */
		if (pvmw->pte)
			return not_found(pvmw);
		/*
		 * All callers that get here will already hold the
		 * i_mmap_rwsem.  Therefore, no additional locks need to be
		 * taken before calling hugetlb_walk().
		 */
		pvmw->pte = hugetlb_walk(vma, pvmw->address, size);
		if (!pvmw->pte)
			return false;

		pvmw->ptl = huge_pte_lock(hstate, mm, pvmw->pte);
		if (!check_pte(pvmw))
			return not_found(pvmw);
		return true;
	}

	end = vma_address_end(pvmw);
	if (pvmw->pte)
		goto next_pte;
restart:
	do {
		pgd = pgd_offset(mm, pvmw->address);
		if (!pgd_present(*pgd)) {
			step_forward(pvmw, PGDIR_SIZE);
			continue;
		}
		p4d = p4d_offset(pgd, pvmw->address);
		if (!p4d_present(*p4d)) {
			step_forward(pvmw, P4D_SIZE);
			continue;
		}
		pud = pud_offset(p4d, pvmw->address);
		if (!pud_present(*pud)) {
			step_forward(pvmw, PUD_SIZE);
			continue;
		}

		pvmw->pmd = pmd_offset(pud, pvmw->address);
		/*
		 * Make sure the pmd value isn't cached in a register by the
		 * compiler and used as a stale value after we've observed a
		 * subsequent update.
		 */
		pmde = pmdp_get_lockless(pvmw->pmd);

		if (pmd_trans_huge(pmde) || is_pmd_migration_entry(pmde) ||
		    (pmd_present(pmde) && pmd_devmap(pmde))) {
			pvmw->ptl = pmd_lock(mm, pvmw->pmd);
			pmde = *pvmw->pmd;
			if (!pmd_present(pmde)) {
				swp_entry_t entry;

				if (!thp_migration_supported() ||
				    !(pvmw->flags & PVMW_MIGRATION))
					return not_found(pvmw);
				entry = pmd_to_swp_entry(pmde);
				if (!is_migration_entry(entry) ||
				    !check_pmd(swp_offset_pfn(entry), pvmw))
					return not_found(pvmw);
				return true;
			}
			if (likely(pmd_trans_huge(pmde) || pmd_devmap(pmde))) {
				if (pvmw->flags & PVMW_MIGRATION)
					return not_found(pvmw);
				if (!check_pmd(pmd_pfn(pmde), pvmw))
					return not_found(pvmw);
				return true;
			}
			/* THP pmd was split under us: handle on pte level */
			spin_unlock(pvmw->ptl);
			pvmw->ptl = NULL;
		} else if (!pmd_present(pmde)) {
			/*
			 * If PVMW_SYNC, take and drop THP pmd lock so that we
			 * cannot return prematurely, while zap_huge_pmd() has
			 * cleared *pmd but not decremented compound_mapcount().
			 */
			if ((pvmw->flags & PVMW_SYNC) &&
			    transhuge_vma_suitable(vma, pvmw->address) &&
			    (pvmw->nr_pages >= HPAGE_PMD_NR)) {
				spinlock_t *ptl = pmd_lock(mm, pvmw->pmd);

				spin_unlock(ptl);
			}
			step_forward(pvmw, PMD_SIZE);
			continue;
		}
		if (!map_pte(pvmw, &ptl)) {
			if (!pvmw->pte)
				goto restart;
			goto next_pte;
		}
this_pte:
		if (check_pte(pvmw))
			return true;
next_pte:
		do {
			pvmw->address += PAGE_SIZE;
			if (pvmw->address >= end)
				return not_found(pvmw);
			/* Did we cross page table boundary? */
			if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
				if (pvmw->ptl) {
					spin_unlock(pvmw->ptl);
					pvmw->ptl = NULL;
				}
				pte_unmap(pvmw->pte);
				pvmw->pte = NULL;
				goto restart;
			}
			pvmw->pte++;
		} while (pte_none(ptep_get(pvmw->pte)));

		if (!pvmw->ptl) {
			pvmw->ptl = ptl;
			spin_lock(pvmw->ptl);
		}
		goto this_pte;
	} while (pvmw->address < end);

	return false;
}

/**
 * page_mapped_in_vma - check whether a page is really mapped in a VMA
 * @page: the page to test
 * @vma: the VMA to test
 *
 * Returns 1 if the page is mapped into the page tables of the VMA, 0
 * if the page is not mapped into the page tables of this VMA.  Only
 * valid for normal file or anonymous VMAs.
 */
int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
{
	struct page_vma_mapped_walk pvmw = {
		.pfn = page_to_pfn(page),
		.nr_pages = 1,
		.vma = vma,
		.flags = PVMW_SYNC,
	};

	pvmw.address = vma_address(page, vma);
	if (pvmw.address == -EFAULT)
		return 0;
	if (!page_vma_mapped_walk(&pvmw))
		return 0;
	page_vma_mapped_walk_done(&pvmw);
	return 1;
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
ace71a19 KS	2	#include <linux/mm.h>
	3	#include <linux/rmap.h>
	4	#include <linux/hugetlb.h>
	5	#include <linux/swap.h>
	6	#include <linux/swapops.h>
	7
	8	#include "internal.h"
	9
ace71a19 KS	10	static inline bool not_found(struct page_vma_mapped_walk *pvmw)
	11	{
	12	page_vma_mapped_walk_done(pvmw);
	13	return false;
	14	}
	15
2798bbe7	16	static bool map_pte(struct page_vma_mapped_walk pvmw, spinlock_t *ptlp)
ace71a19	17	{
c33c7948 RR	18	pte_t ptent;
c33c7948 RR	19
90f43b0a HD	20	if (pvmw->flags & PVMW_SYNC) {
	21	/* Use the stricter lookup */
	22	pvmw->pte = pte_offset_map_lock(pvmw->vma->vm_mm, pvmw->pmd,
	23	pvmw->address, &pvmw->ptl);
2798bbe7 HD	24	*ptlp = pvmw->ptl;
2798bbe7 HD	25	return !!pvmw->pte;
90f43b0a	26	}
aab8d052	27
2798bbe7 HD	28	/*
	29	* It is important to return the ptl corresponding to pte,
	30	* in case *pvmw->pmd changes underneath us; so we need to
	31	* return it even when choosing not to lock, in case caller
	32	* proceeds to loop over next ptes, and finds a match later.
	33	* Though, in most cases, page lock already protects this.
	34	*/
	35	pvmw->pte = pte_offset_map_nolock(pvmw->vma->vm_mm, pvmw->pmd,
	36	pvmw->address, ptlp);
	37	if (!pvmw->pte)
	38	return false;
	39
c33c7948 RR	40	ptent = ptep_get(pvmw->pte);
c33c7948 RR	41
90f43b0a	42	if (pvmw->flags & PVMW_MIGRATION) {
c33c7948	43	if (!is_swap_pte(ptent))
90f43b0a	44	return false;
c33c7948	45	} else if (is_swap_pte(ptent)) {
90f43b0a HD	46	swp_entry_t entry;
	47	/*
	48	* Handle un-addressable ZONE_DEVICE memory.
	49	*
	50	* We get here when we are trying to unmap a private
	51	* device page from the process address space. Such
	52	* page is not CPU accessible and thus is mapped as
	53	* a special swap entry, nonetheless it still does
	54	* count as a valid regular mapping for the page
	55	* (and is accounted as such in page maps count).
	56	*
	57	* So handle this special case as if it was a normal
	58	* page mapping ie lock CPU page table and return true.
	59	*
	60	* For more details on device private memory see HMM
	61	* (include/linux/hmm.h or mm/hmm.c).
	62	*/
c33c7948	63	entry = pte_to_swp_entry(ptent);
90f43b0a HD	64	if (!is_device_private_entry(entry) &&
	65	!is_device_exclusive_entry(entry))
	66	return false;
c33c7948	67	} else if (!pte_present(ptent)) {
90f43b0a	68	return false;
ace71a19	69	}
2798bbe7	70	pvmw->ptl = *ptlp;
ace71a19 KS	71	spin_lock(pvmw->ptl);
	72	return true;
	73	}
	74
0d665e7b	75	/**
9651eeab KS	76	* check_pte - check if [pvmw->pfn, @pvmw->pfn + @pvmw->nr_pages) is
	77	* mapped at the @pvmw->pte
	78	* @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range
	79	* for checking
0d665e7b	80	*
9651eeab	81	* page_vma_mapped_walk() found a place where pfn range is potentially
0d665e7b KS	82	* mapped. check_pte() has to validate this.
0d665e7b KS	83	*
777f303c AS	84	* pvmw->pte may point to empty PTE, swap PTE or PTE pointing to
777f303c AS	85	* arbitrary page.
0d665e7b KS	86	*
0d665e7b KS	87	* If PVMW_MIGRATION flag is set, returns true if @pvmw->pte contains migration
9651eeab	88	* entry that points to [pvmw->pfn, @pvmw->pfn + @pvmw->nr_pages)
0d665e7b	89	*
777f303c	90	* If PVMW_MIGRATION flag is not set, returns true if pvmw->pte points to
9651eeab	91	* [pvmw->pfn, @pvmw->pfn + @pvmw->nr_pages)
0d665e7b KS	92	*
	93	* Otherwise, return false.
	94	*
	95	*/
ace71a19 KS	96	static bool check_pte(struct page_vma_mapped_walk *pvmw)
ace71a19 KS	97	{
0d665e7b	98	unsigned long pfn;
c33c7948	99	pte_t ptent = ptep_get(pvmw->pte);
0d665e7b	100
ace71a19	101	if (pvmw->flags & PVMW_MIGRATION) {
ace71a19	102	swp_entry_t entry;
c33c7948	103	if (!is_swap_pte(ptent))
ace71a19	104	return false;
c33c7948	105	entry = pte_to_swp_entry(ptent);
a5430dda	106
b756a3b5 AP	107	if (!is_migration_entry(entry) &&
b756a3b5 AP	108	!is_device_exclusive_entry(entry))
ace71a19	109	return false;
a5430dda	110
0d206b5d	111	pfn = swp_offset_pfn(entry);
c33c7948	112	} else if (is_swap_pte(ptent)) {
0d665e7b	113	swp_entry_t entry;
a5430dda	114
0d665e7b	115	/* Handle un-addressable ZONE_DEVICE memory */
c33c7948	116	entry = pte_to_swp_entry(ptent);
b756a3b5 AP	117	if (!is_device_private_entry(entry) &&
b756a3b5 AP	118	!is_device_exclusive_entry(entry))
ace71a19 KS	119	return false;
ace71a19 KS	120
0d206b5d	121	pfn = swp_offset_pfn(entry);
0d665e7b	122	} else {
c33c7948	123	if (!pte_present(ptent))
ace71a19	124	return false;
0d665e7b	125
c33c7948	126	pfn = pte_pfn(ptent);
ace71a19 KS	127	}
ace71a19 KS	128
2aff7a47 MWO	129	return (pfn - pvmw->pfn) < pvmw->nr_pages;
	130	}
	131
	132	/* Returns true if the two ranges overlap. Careful to not overflow. */
	133	static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
	134	{
	135	if ((pfn + HPAGE_PMD_NR - 1) < pvmw->pfn)
	136	return false;
	137	if (pfn > pvmw->pfn + pvmw->nr_pages - 1)
	138	return false;
	139	return true;
ace71a19 KS	140	}
ace71a19 KS	141
a9a7504d HD	142	static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
	143	{
	144	pvmw->address = (pvmw->address + size) & ~(size - 1);
	145	if (!pvmw->address)
	146	pvmw->address = ULONG_MAX;
	147	}
	148
ace71a19	149	/**
2aff7a47	150	* page_vma_mapped_walk - check if @pvmw->pfn is mapped in @pvmw->vma at
ace71a19 KS	151	* @pvmw->address
	152	* @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
	153	* must be set. pmd, pte and ptl must be NULL.
	154	*
	155	* Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point
	156	* to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is
	157	* adjusted if needed (for PTE-mapped THPs).
	158	*
	159	* If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page
	160	* (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in
	161	* a loop to find all PTEs that map the THP.
	162	*
	163	* For HugeTLB pages, @pvmw->pte is set to the relevant page table entry
	164	* regardless of which page table level the page is mapped at. @pvmw->pmd is
	165	* NULL.
	166	*
baf2f90b	167	* Returns false if there are no more page table entries for the page in
ace71a19 KS	168	* the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped.
	169	*
	170	* If you need to stop the walk before page_vma_mapped_walk() returned false,
	171	* use page_vma_mapped_walk_done(). It will do the housekeeping.
	172	*/
	173	bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
	174	{
2aff7a47 MWO	175	struct vm_area_struct *vma = pvmw->vma;
2aff7a47 MWO	176	struct mm_struct *mm = vma->vm_mm;
47446630	177	unsigned long end;
2798bbe7	178	spinlock_t *ptl;
ace71a19	179	pgd_t *pgd;
c2febafc	180	p4d_t *p4d;
ace71a19	181	pud_t *pud;
a7b10095	182	pmd_t pmde;
ace71a19 KS	183
	184	/* The only possible pmd mapping has been handled on last iteration */
	185	if (pvmw->pmd && !pvmw->pte)
	186	return not_found(pvmw);
	187
2aff7a47	188	if (unlikely(is_vm_hugetlb_page(vma))) {
98ea0259	189	struct hstate *hstate = hstate_vma(vma);
98ea0259	190	unsigned long size = huge_page_size(hstate);
6d0fd598 HD	191	/* The only possible mapping was handled on last iteration */
	192	if (pvmw->pte)
	193	return not_found(pvmw);
9c67a207 PX	194	/*
	195	* All callers that get here will already hold the
	196	* i_mmap_rwsem. Therefore, no additional locks need to be
	197	* taken before calling hugetlb_walk().
	198	*/
	199	pvmw->pte = hugetlb_walk(vma, pvmw->address, size);
ace71a19 KS	200	if (!pvmw->pte)
	201	return false;
	202
8f0b747d	203	pvmw->ptl = huge_pte_lock(hstate, mm, pvmw->pte);
ace71a19 KS	204	if (!check_pte(pvmw))
	205	return not_found(pvmw);
	206	return true;
	207	}
6d0fd598	208
2aff7a47	209	end = vma_address_end(pvmw);
6d0fd598 HD	210	if (pvmw->pte)
6d0fd598 HD	211	goto next_pte;
ace71a19	212	restart:
a9a7504d	213	do {
b3807a91	214	pgd = pgd_offset(mm, pvmw->address);
a9a7504d HD	215	if (!pgd_present(*pgd)) {
	216	step_forward(pvmw, PGDIR_SIZE);
	217	continue;
	218	}
b3807a91	219	p4d = p4d_offset(pgd, pvmw->address);
a9a7504d HD	220	if (!p4d_present(*p4d)) {
	221	step_forward(pvmw, P4D_SIZE);
	222	continue;
	223	}
b3807a91	224	pud = pud_offset(p4d, pvmw->address);
a9a7504d HD	225	if (!pud_present(*pud)) {
	226	step_forward(pvmw, PUD_SIZE);
	227	continue;
	228	}
e2e1d407	229
b3807a91	230	pvmw->pmd = pmd_offset(pud, pvmw->address);
732ed558	231	/*
b3807a91 HD	232	* Make sure the pmd value isn't cached in a register by the
	233	* compiler and used as a stale value after we've observed a
	234	* subsequent update.
732ed558	235	*/
26e1a0c3	236	pmde = pmdp_get_lockless(pvmw->pmd);
b3807a91	237
6472f6d2 MS	238	if (pmd_trans_huge(pmde) \|\| is_pmd_migration_entry(pmde) \|\|
6472f6d2 MS	239	(pmd_present(pmde) && pmd_devmap(pmde))) {
b3807a91 HD	240	pvmw->ptl = pmd_lock(mm, pvmw->pmd);
b3807a91 HD	241	pmde = *pvmw->pmd;
b3807a91 HD	242	if (!pmd_present(pmde)) {
	243	swp_entry_t entry;
	244
	245	if (!thp_migration_supported() \|\|
	246	!(pvmw->flags & PVMW_MIGRATION))
	247	return not_found(pvmw);
	248	entry = pmd_to_swp_entry(pmde);
	249	if (!is_migration_entry(entry) \|\|
0d206b5d	250	!check_pmd(swp_offset_pfn(entry), pvmw))
b3807a91 HD	251	return not_found(pvmw);
	252	return true;
	253	}
6472f6d2 MS	254	if (likely(pmd_trans_huge(pmde) \|\| pmd_devmap(pmde))) {
	255	if (pvmw->flags & PVMW_MIGRATION)
	256	return not_found(pvmw);
	257	if (!check_pmd(pmd_pfn(pmde), pvmw))
	258	return not_found(pvmw);
	259	return true;
	260	}
b3807a91 HD	261	/* THP pmd was split under us: handle on pte level */
	262	spin_unlock(pvmw->ptl);
	263	pvmw->ptl = NULL;
	264	} else if (!pmd_present(pmde)) {
	265	/*
	266	* If PVMW_SYNC, take and drop THP pmd lock so that we
	267	* cannot return prematurely, while zap_huge_pmd() has
	268	* cleared *pmd but not decremented compound_mapcount().
	269	*/
	270	if ((pvmw->flags & PVMW_SYNC) &&
c453d8c7	271	transhuge_vma_suitable(vma, pvmw->address) &&
2aff7a47	272	(pvmw->nr_pages >= HPAGE_PMD_NR)) {
b3807a91	273	spinlock_t *ptl = pmd_lock(mm, pvmw->pmd);
732ed558	274
b3807a91 HD	275	spin_unlock(ptl);
b3807a91 HD	276	}
a9a7504d HD	277	step_forward(pvmw, PMD_SIZE);
a9a7504d HD	278	continue;
732ed558	279	}
2798bbe7 HD	280	if (!map_pte(pvmw, &ptl)) {
	281	if (!pvmw->pte)
	282	goto restart;
b3807a91	283	goto next_pte;
2798bbe7	284	}
47446630	285	this_pte:
ace71a19 KS	286	if (check_pte(pvmw))
ace71a19 KS	287	return true;
d75450ff	288	next_pte:
d75450ff	289	do {
ace71a19	290	pvmw->address += PAGE_SIZE;
494334e4	291	if (pvmw->address >= end)
ace71a19 KS	292	return not_found(pvmw);
ace71a19 KS	293	/* Did we cross page table boundary? */
44828248	294	if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
ace71a19 KS	295	if (pvmw->ptl) {
	296	spin_unlock(pvmw->ptl);
	297	pvmw->ptl = NULL;
	298	}
44828248 HD	299	pte_unmap(pvmw->pte);
44828248 HD	300	pvmw->pte = NULL;
ace71a19	301	goto restart;
ace71a19	302	}
44828248	303	pvmw->pte++;
c33c7948	304	} while (pte_none(ptep_get(pvmw->pte)));
ace71a19 KS	305
ace71a19 KS	306	if (!pvmw->ptl) {
2798bbe7	307	pvmw->ptl = ptl;
ace71a19 KS	308	spin_lock(pvmw->ptl);
ace71a19 KS	309	}
47446630	310	goto this_pte;
a9a7504d HD	311	} while (pvmw->address < end);
	312
	313	return false;
ace71a19	314	}
6a328a62 KS	315
	316	/**
	317	* page_mapped_in_vma - check whether a page is really mapped in a VMA
	318	* @page: the page to test
	319	* @vma: the VMA to test
	320	*
	321	* Returns 1 if the page is mapped into the page tables of the VMA, 0
	322	* if the page is not mapped into the page tables of this VMA. Only
	323	* valid for normal file or anonymous VMAs.
	324	*/
	325	int page_mapped_in_vma(struct page page, struct vm_area_struct vma)
	326	{
	327	struct page_vma_mapped_walk pvmw = {
2aff7a47 MWO	328	.pfn = page_to_pfn(page),
2aff7a47 MWO	329	.nr_pages = 1,
6a328a62 KS	330	.vma = vma,
	331	.flags = PVMW_SYNC,
	332	};
6a328a62	333
494334e4 HD	334	pvmw.address = vma_address(page, vma);
494334e4 HD	335	if (pvmw.address == -EFAULT)
6a328a62	336	return 0;
6a328a62 KS	337	if (!page_vma_mapped_walk(&pvmw))
	338	return 0;
	339	page_vma_mapped_walk_done(&pvmw);
	340	return 1;
	341	}