[thirdparty/kernel/linux.git] / arch / x86 / mm / pgtable.c

#include <linux/mm.h>
#include <linux/gfp.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/tlb.h>
#include <asm/fixmap.h>

#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO

#ifdef CONFIG_HIGHPTE
#define PGALLOC_USER_GFP __GFP_HIGHMEM
#else
#define PGALLOC_USER_GFP 0
#endif

gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP;

pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
	return (pte_t *)__get_free_page(PGALLOC_GFP);
}

pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	struct page *pte;

	pte = alloc_pages(__userpte_alloc_gfp, 0);
	if (!pte)
		return NULL;
	if (!pgtable_page_ctor(pte)) {
		__free_page(pte);
		return NULL;
	}
	return pte;
}

static int __init setup_userpte(char *arg)
{
	if (!arg)
		return -EINVAL;

	/*
	 * "userpte=nohigh" disables allocation of user pagetables in
	 * high memory.
	 */
	if (strcmp(arg, "nohigh") == 0)
		__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
	else
		return -EINVAL;
	return 0;
}
early_param("userpte", setup_userpte);

void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
	pgtable_page_dtor(pte);
	paravirt_release_pte(page_to_pfn(pte));
	tlb_remove_page(tlb, pte);
}

#if PAGETABLE_LEVELS > 2
void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
{
	struct page *page = virt_to_page(pmd);
	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
	/*
	 * NOTE! For PAE, any changes to the top page-directory-pointer-table
	 * entries need a full cr3 reload to flush.
	 */
#ifdef CONFIG_X86_PAE
	tlb->need_flush_all = 1;
#endif
	pgtable_pmd_page_dtor(page);
	tlb_remove_page(tlb, page);
}

#if PAGETABLE_LEVELS > 3
void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{
	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
	tlb_remove_page(tlb, virt_to_page(pud));
}
#endif	/* PAGETABLE_LEVELS > 3 */
#endif	/* PAGETABLE_LEVELS > 2 */

static inline void pgd_list_add(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_add(&page->lru, &pgd_list);
}

static inline void pgd_list_del(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_del(&page->lru);
}

#define UNSHARED_PTRS_PER_PGD				\
	(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)


static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
{
	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
	virt_to_page(pgd)->index = (pgoff_t)mm;
}

struct mm_struct *pgd_page_get_mm(struct page *page)
{
	return (struct mm_struct *)page->index;
}

static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
{
	/* If the pgd points to a shared pagetable level (either the
	   ptes in non-PAE, or shared PMD in PAE), then just copy the
	   references from swapper_pg_dir. */
	if (PAGETABLE_LEVELS == 2 ||
	    (PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) ||
	    PAGETABLE_LEVELS == 4) {
		clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
				swapper_pg_dir + KERNEL_PGD_BOUNDARY,
				KERNEL_PGD_PTRS);
	}

	/* list required to sync kernel mapping updates */
	if (!SHARED_KERNEL_PMD) {
		pgd_set_mm(pgd, mm);
		pgd_list_add(pgd);
	}
}

static void pgd_dtor(pgd_t *pgd)
{
	if (SHARED_KERNEL_PMD)
		return;

	spin_lock(&pgd_lock);
	pgd_list_del(pgd);
	spin_unlock(&pgd_lock);
}

/*
 * List of all pgd's needed for non-PAE so it can invalidate entries
 * in both cached and uncached pgd's; not needed for PAE since the
 * kernel pmd is shared. If PAE were not to share the pmd a similar
 * tactic would be needed. This is essentially codepath-based locking
 * against pageattr.c; it is the unique case in which a valid change
 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
 * vmalloc faults work because attached pagetables are never freed.
 * -- nyc
 */

#ifdef CONFIG_X86_PAE
/*
 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
 * updating the top-level pagetable entries to guarantee the
 * processor notices the update.  Since this is expensive, and
 * all 4 top-level entries are used almost immediately in a
 * new process's life, we just pre-populate them here.
 *
 * Also, if we're in a paravirt environment where the kernel pmd is
 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
 * and initialize the kernel pmds here.
 */
#define PREALLOCATED_PMDS	UNSHARED_PTRS_PER_PGD

void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
{
	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);

	/* Note: almost everything apart from _PAGE_PRESENT is
	   reserved at the pmd (PDPT) level. */
	set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));

	/*
	 * According to Intel App note "TLBs, Paging-Structure Caches,
	 * and Their Invalidation", April 2007, document 317080-001,
	 * section 8.1: in PAE mode we explicitly have to flush the
	 * TLB via cr3 if the top-level pgd is changed...
	 */
	flush_tlb_mm(mm);
}
#else  /* !CONFIG_X86_PAE */

/* No need to prepopulate any pagetable entries in non-PAE modes. */
#define PREALLOCATED_PMDS	0

#endif	/* CONFIG_X86_PAE */

static void free_pmds(pmd_t *pmds[])
{
	int i;

	for(i = 0; i < PREALLOCATED_PMDS; i++)
		if (pmds[i]) {
			pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
			free_page((unsigned long)pmds[i]);
		}
}

static int preallocate_pmds(pmd_t *pmds[])
{
	int i;
	bool failed = false;

	for(i = 0; i < PREALLOCATED_PMDS; i++) {
		pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP);
		if (!pmd)
			failed = true;
		if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
			free_page((unsigned long)pmd);
			pmd = NULL;
			failed = true;
		}
		pmds[i] = pmd;
	}

	if (failed) {
		free_pmds(pmds);
		return -ENOMEM;
	}

	return 0;
}

/*
 * Mop up any pmd pages which may still be attached to the pgd.
 * Normally they will be freed by munmap/exit_mmap, but any pmd we
 * preallocate which never got a corresponding vma will need to be
 * freed manually.
 */
static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
{
	int i;

	for(i = 0; i < PREALLOCATED_PMDS; i++) {
		pgd_t pgd = pgdp[i];

		if (pgd_val(pgd) != 0) {
			pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);

			pgdp[i] = native_make_pgd(0);

			paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
			pmd_free(mm, pmd);
		}
	}
}

static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
{
	pud_t *pud;
	int i;

	if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
		return;

	pud = pud_offset(pgd, 0);

	for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
		pmd_t *pmd = pmds[i];

		if (i >= KERNEL_PGD_BOUNDARY)
			memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
			       sizeof(pmd_t) * PTRS_PER_PMD);

		pud_populate(mm, pud, pmd);
	}
}

pgd_t *pgd_alloc(struct mm_struct *mm)
{
	pgd_t *pgd;
	pmd_t *pmds[PREALLOCATED_PMDS];

	pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);

	if (pgd == NULL)
		goto out;

	mm->pgd = pgd;

	if (preallocate_pmds(pmds) != 0)
		goto out_free_pgd;

	if (paravirt_pgd_alloc(mm) != 0)
		goto out_free_pmds;

	/*
	 * Make sure that pre-populating the pmds is atomic with
	 * respect to anything walking the pgd_list, so that they
	 * never see a partially populated pgd.
	 */
	spin_lock(&pgd_lock);

	pgd_ctor(mm, pgd);
	pgd_prepopulate_pmd(mm, pgd, pmds);

	spin_unlock(&pgd_lock);

	return pgd;

out_free_pmds:
	free_pmds(pmds);
out_free_pgd:
	free_page((unsigned long)pgd);
out:
	return NULL;
}

void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
	pgd_mop_up_pmds(mm, pgd);
	pgd_dtor(pgd);
	paravirt_pgd_free(mm, pgd);
	free_page((unsigned long)pgd);
}

/*
 * Used to set accessed or dirty bits in the page table entries
 * on other architectures. On x86, the accessed and dirty bits
 * are tracked by hardware. However, do_wp_page calls this function
 * to also make the pte writeable at the same time the dirty bit is
 * set. In that case we do actually need to write the PTE.
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
	int changed = !pte_same(*ptep, entry);

	if (changed && dirty) {
		*ptep = entry;
		pte_update_defer(vma->vm_mm, address, ptep);
	}

	return changed;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp,
			  pmd_t entry, int dirty)
{
	int changed = !pmd_same(*pmdp, entry);

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	if (changed && dirty) {
		*pmdp = entry;
		pmd_update_defer(vma->vm_mm, address, pmdp);
		/*
		 * We had a write-protection fault here and changed the pmd
		 * to to more permissive. No need to flush the TLB for that,
		 * #PF is architecturally guaranteed to do that and in the
		 * worst-case we'll generate a spurious fault.
		 */
	}

	return changed;
}
#endif

int ptep_test_and_clear_young(struct vm_area_struct *vma,
			      unsigned long addr, pte_t *ptep)
{
	int ret = 0;

	if (pte_young(*ptep))
		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
					 (unsigned long *) &ptep->pte);

	if (ret)
		pte_update(vma->vm_mm, addr, ptep);

	return ret;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
			      unsigned long addr, pmd_t *pmdp)
{
	int ret = 0;

	if (pmd_young(*pmdp))
		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
					 (unsigned long *)pmdp);

	if (ret)
		pmd_update(vma->vm_mm, addr, pmdp);

	return ret;
}
#endif

int ptep_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pte_t *ptep)
{
	int young;

	young = ptep_test_and_clear_young(vma, address, ptep);
	if (young)
		flush_tlb_page(vma, address);

	return young;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pmd_t *pmdp)
{
	int young;

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	young = pmdp_test_and_clear_young(vma, address, pmdp);
	if (young)
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);

	return young;
}

void pmdp_splitting_flush(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp)
{
	int set;
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
				(unsigned long *)pmdp);
	if (set) {
		pmd_update(vma->vm_mm, address, pmdp);
		/* need tlb flush only to serialize against gup-fast */
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}
}
#endif

/**
 * reserve_top_address - reserves a hole in the top of kernel address space
 * @reserve - size of hole to reserve
 *
 * Can be used to relocate the fixmap area and poke a hole in the top
 * of kernel address space to make room for a hypervisor.
 */
void __init reserve_top_address(unsigned long reserve)
{
#ifdef CONFIG_X86_32
	BUG_ON(fixmaps_set > 0);
	printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
	       (int)-reserve);
	__FIXADDR_TOP = -reserve - PAGE_SIZE;
#endif
}

int fixmaps_set;

void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
{
	unsigned long address = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		BUG();
		return;
	}
	set_pte_vaddr(address, pte);
	fixmaps_set++;
}

void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
		       pgprot_t flags)
{
	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
}
Commit	Line	Data
4f76cd38	1	#include <linux/mm.h>
5a0e3ad6	2	#include <linux/gfp.h>
4f76cd38	3	#include <asm/pgalloc.h>
ee5aa8d3	4	#include <asm/pgtable.h>
4f76cd38	5	#include <asm/tlb.h>
a1d5a869	6	#include <asm/fixmap.h>
4f76cd38	7
9e730237 VN	8	#define PGALLOC_GFP GFP_KERNEL \| __GFP_NOTRACK \| __GFP_REPEAT \| __GFP_ZERO
9e730237 VN	9
14315592 IC	10	#ifdef CONFIG_HIGHPTE
	11	#define PGALLOC_USER_GFP __GFP_HIGHMEM
	12	#else
	13	#define PGALLOC_USER_GFP 0
	14	#endif
	15
	16	gfp_t __userpte_alloc_gfp = PGALLOC_GFP \| PGALLOC_USER_GFP;
	17
4f76cd38 JF	18	pte_t pte_alloc_one_kernel(struct mm_struct mm, unsigned long address)
4f76cd38 JF	19	{
9e730237	20	return (pte_t *)__get_free_page(PGALLOC_GFP);
4f76cd38 JF	21	}
	22
	23	pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
	24	{
	25	struct page *pte;
	26
14315592	27	pte = alloc_pages(__userpte_alloc_gfp, 0);
cecbd1b5 KS	28	if (!pte)
	29	return NULL;
	30	if (!pgtable_page_ctor(pte)) {
	31	__free_page(pte);
	32	return NULL;
	33	}
4f76cd38 JF	34	return pte;
	35	}
	36
14315592 IC	37	static int __init setup_userpte(char *arg)
	38	{
	39	if (!arg)
	40	return -EINVAL;
	41
	42	/*
	43	* "userpte=nohigh" disables allocation of user pagetables in
	44	* high memory.
	45	*/
	46	if (strcmp(arg, "nohigh") == 0)
	47	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
	48	else
	49	return -EINVAL;
	50	return 0;
	51	}
	52	early_param("userpte", setup_userpte);
	53
9e1b32ca	54	void ___pte_free_tlb(struct mmu_gather tlb, struct page pte)
397f687a JF	55	{
397f687a JF	56	pgtable_page_dtor(pte);
6944a9c8	57	paravirt_release_pte(page_to_pfn(pte));
397f687a JF	58	tlb_remove_page(tlb, pte);
	59	}
	60
170fdff7	61	#if PAGETABLE_LEVELS > 2
9e1b32ca	62	void ___pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd)
170fdff7	63	{
c283610e	64	struct page *page = virt_to_page(pmd);
6944a9c8	65	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
1de14c3c DH	66	/*
	67	* NOTE! For PAE, any changes to the top page-directory-pointer-table
	68	* entries need a full cr3 reload to flush.
	69	*/
	70	#ifdef CONFIG_X86_PAE
	71	tlb->need_flush_all = 1;
	72	#endif
c283610e KS	73	pgtable_pmd_page_dtor(page);
c283610e KS	74	tlb_remove_page(tlb, page);
170fdff7	75	}
5a5f8f42 JF	76
5a5f8f42 JF	77	#if PAGETABLE_LEVELS > 3
9e1b32ca	78	void ___pud_free_tlb(struct mmu_gather tlb, pud_t pud)
5a5f8f42	79	{
2761fa09	80	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
5a5f8f42 JF	81	tlb_remove_page(tlb, virt_to_page(pud));
	82	}
	83	#endif /* PAGETABLE_LEVELS > 3 */
170fdff7 JF	84	#endif /* PAGETABLE_LEVELS > 2 */
170fdff7 JF	85
4f76cd38 JF	86	static inline void pgd_list_add(pgd_t *pgd)
	87	{
	88	struct page *page = virt_to_page(pgd);
4f76cd38	89
4f76cd38	90	list_add(&page->lru, &pgd_list);
4f76cd38 JF	91	}
	92
	93	static inline void pgd_list_del(pgd_t *pgd)
	94	{
	95	struct page *page = virt_to_page(pgd);
4f76cd38	96
4f76cd38	97	list_del(&page->lru);
4f76cd38 JF	98	}
4f76cd38 JF	99
4f76cd38	100	#define UNSHARED_PTRS_PER_PGD \
68db065c	101	(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
4f76cd38	102
617d34d9 JF	103
	104	static void pgd_set_mm(pgd_t pgd, struct mm_struct mm)
	105	{
	106	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
	107	virt_to_page(pgd)->index = (pgoff_t)mm;
	108	}
	109
	110	struct mm_struct pgd_page_get_mm(struct page page)
	111	{
	112	return (struct mm_struct *)page->index;
	113	}
	114
	115	static void pgd_ctor(struct mm_struct mm, pgd_t pgd)
4f76cd38	116	{
4f76cd38 JF	117	/* If the pgd points to a shared pagetable level (either the
	118	ptes in non-PAE, or shared PMD in PAE), then just copy the
	119	references from swapper_pg_dir. */
	120	if (PAGETABLE_LEVELS == 2 \|\|
85958b46 JF	121	(PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) \|\|
85958b46 JF	122	PAGETABLE_LEVELS == 4) {
68db065c JF	123	clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
68db065c JF	124	swapper_pg_dir + KERNEL_PGD_BOUNDARY,
4f76cd38	125	KERNEL_PGD_PTRS);
4f76cd38 JF	126	}
	127
	128	/* list required to sync kernel mapping updates */
617d34d9 JF	129	if (!SHARED_KERNEL_PMD) {
617d34d9 JF	130	pgd_set_mm(pgd, mm);
4f76cd38	131	pgd_list_add(pgd);
617d34d9	132	}
4f76cd38 JF	133	}
4f76cd38 JF	134
17b74627	135	static void pgd_dtor(pgd_t *pgd)
4f76cd38	136	{
4f76cd38 JF	137	if (SHARED_KERNEL_PMD)
	138	return;
	139
a79e53d8	140	spin_lock(&pgd_lock);
4f76cd38	141	pgd_list_del(pgd);
a79e53d8	142	spin_unlock(&pgd_lock);
4f76cd38 JF	143	}
4f76cd38 JF	144
85958b46 JF	145	/*
	146	* List of all pgd's needed for non-PAE so it can invalidate entries
	147	* in both cached and uncached pgd's; not needed for PAE since the
	148	* kernel pmd is shared. If PAE were not to share the pmd a similar
	149	* tactic would be needed. This is essentially codepath-based locking
	150	* against pageattr.c; it is the unique case in which a valid change
	151	* of kernel pagetables can't be lazily synchronized by vmalloc faults.
	152	* vmalloc faults work because attached pagetables are never freed.
6d49e352	153	* -- nyc
85958b46 JF	154	*/
85958b46 JF	155
4f76cd38	156	#ifdef CONFIG_X86_PAE
d8d5900e JF	157	/*
	158	* In PAE mode, we need to do a cr3 reload (=tlb flush) when
	159	* updating the top-level pagetable entries to guarantee the
	160	* processor notices the update. Since this is expensive, and
	161	* all 4 top-level entries are used almost immediately in a
	162	* new process's life, we just pre-populate them here.
	163	*
	164	* Also, if we're in a paravirt environment where the kernel pmd is
	165	* not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
	166	* and initialize the kernel pmds here.
	167	*/
	168	#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
	169
	170	void pud_populate(struct mm_struct mm, pud_t pudp, pmd_t *pmd)
	171	{
	172	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
	173
	174	/* Note: almost everything apart from _PAGE_PRESENT is
	175	reserved at the pmd (PDPT) level. */
	176	set_pud(pudp, __pud(__pa(pmd) \| _PAGE_PRESENT));
	177
	178	/*
	179	* According to Intel App note "TLBs, Paging-Structure Caches,
	180	* and Their Invalidation", April 2007, document 317080-001,
	181	* section 8.1: in PAE mode we explicitly have to flush the
	182	* TLB via cr3 if the top-level pgd is changed...
	183	*/
4981d01e	184	flush_tlb_mm(mm);
d8d5900e JF	185	}
	186	#else /* !CONFIG_X86_PAE */
	187
	188	/* No need to prepopulate any pagetable entries in non-PAE modes. */
	189	#define PREALLOCATED_PMDS 0
	190
	191	#endif /* CONFIG_X86_PAE */
	192
	193	static void free_pmds(pmd_t *pmds[])
	194	{
	195	int i;
	196
	197	for(i = 0; i < PREALLOCATED_PMDS; i++)
09ef4939 KS	198	if (pmds[i]) {
09ef4939 KS	199	pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
d8d5900e	200	free_page((unsigned long)pmds[i]);
09ef4939	201	}
d8d5900e JF	202	}
	203
	204	static int preallocate_pmds(pmd_t *pmds[])
	205	{
	206	int i;
	207	bool failed = false;
	208
	209	for(i = 0; i < PREALLOCATED_PMDS; i++) {
9e730237	210	pmd_t pmd = (pmd_t )__get_free_page(PGALLOC_GFP);
09ef4939	211	if (!pmd)
d8d5900e	212	failed = true;
09ef4939	213	if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
2a46eed5	214	free_page((unsigned long)pmd);
09ef4939 KS	215	pmd = NULL;
	216	failed = true;
	217	}
d8d5900e JF	218	pmds[i] = pmd;
	219	}
	220
	221	if (failed) {
	222	free_pmds(pmds);
	223	return -ENOMEM;
	224	}
	225
	226	return 0;
	227	}
	228
4f76cd38 JF	229	/*
	230	* Mop up any pmd pages which may still be attached to the pgd.
	231	* Normally they will be freed by munmap/exit_mmap, but any pmd we
	232	* preallocate which never got a corresponding vma will need to be
	233	* freed manually.
	234	*/
	235	static void pgd_mop_up_pmds(struct mm_struct mm, pgd_t pgdp)
	236	{
	237	int i;
	238
d8d5900e	239	for(i = 0; i < PREALLOCATED_PMDS; i++) {
4f76cd38 JF	240	pgd_t pgd = pgdp[i];
	241
	242	if (pgd_val(pgd) != 0) {
	243	pmd_t pmd = (pmd_t )pgd_page_vaddr(pgd);
	244
	245	pgdp[i] = native_make_pgd(0);
	246
6944a9c8	247	paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
4f76cd38 JF	248	pmd_free(mm, pmd);
	249	}
	250	}
	251	}
	252
d8d5900e	253	static void pgd_prepopulate_pmd(struct mm_struct mm, pgd_t pgd, pmd_t *pmds[])
4f76cd38 JF	254	{
4f76cd38 JF	255	pud_t *pud;
4f76cd38 JF	256	int i;
4f76cd38 JF	257
cf3e5050 JF	258	if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
	259	return;
	260
4f76cd38	261	pud = pud_offset(pgd, 0);
4f76cd38	262
73b44ff4	263	for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
d8d5900e	264	pmd_t *pmd = pmds[i];
4f76cd38	265
68db065c	266	if (i >= KERNEL_PGD_BOUNDARY)
4f76cd38 JF	267	memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
	268	sizeof(pmd_t) * PTRS_PER_PMD);
	269
	270	pud_populate(mm, pud, pmd);
	271	}
4f76cd38	272	}
1ec1fe73	273
d8d5900e	274	pgd_t pgd_alloc(struct mm_struct mm)
1ec1fe73	275	{
d8d5900e JF	276	pgd_t *pgd;
d8d5900e JF	277	pmd_t *pmds[PREALLOCATED_PMDS];
1ec1fe73	278
9e730237	279	pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);
d8d5900e JF	280
	281	if (pgd == NULL)
	282	goto out;
	283
	284	mm->pgd = pgd;
	285
	286	if (preallocate_pmds(pmds) != 0)
	287	goto out_free_pgd;
	288
	289	if (paravirt_pgd_alloc(mm) != 0)
	290	goto out_free_pmds;
1ec1fe73 IM	291
1ec1fe73 IM	292	/*
d8d5900e JF	293	* Make sure that pre-populating the pmds is atomic with
	294	* respect to anything walking the pgd_list, so that they
	295	* never see a partially populated pgd.
1ec1fe73	296	*/
a79e53d8	297	spin_lock(&pgd_lock);
4f76cd38	298
617d34d9	299	pgd_ctor(mm, pgd);
d8d5900e	300	pgd_prepopulate_pmd(mm, pgd, pmds);
4f76cd38	301
a79e53d8	302	spin_unlock(&pgd_lock);
4f76cd38 JF	303
4f76cd38 JF	304	return pgd;
d8d5900e JF	305
	306	out_free_pmds:
	307	free_pmds(pmds);
	308	out_free_pgd:
	309	free_page((unsigned long)pgd);
	310	out:
	311	return NULL;
4f76cd38 JF	312	}
	313
	314	void pgd_free(struct mm_struct mm, pgd_t pgd)
	315	{
	316	pgd_mop_up_pmds(mm, pgd);
	317	pgd_dtor(pgd);
eba0045f	318	paravirt_pgd_free(mm, pgd);
4f76cd38 JF	319	free_page((unsigned long)pgd);
4f76cd38 JF	320	}
ee5aa8d3	321
0f9a921c RR	322	/*
	323	* Used to set accessed or dirty bits in the page table entries
	324	* on other architectures. On x86, the accessed and dirty bits
	325	* are tracked by hardware. However, do_wp_page calls this function
	326	* to also make the pte writeable at the same time the dirty bit is
	327	* set. In that case we do actually need to write the PTE.
	328	*/
ee5aa8d3 JF	329	int ptep_set_access_flags(struct vm_area_struct *vma,
	330	unsigned long address, pte_t *ptep,
	331	pte_t entry, int dirty)
	332	{
	333	int changed = !pte_same(*ptep, entry);
	334
	335	if (changed && dirty) {
	336	*ptep = entry;
	337	pte_update_defer(vma->vm_mm, address, ptep);
ee5aa8d3 JF	338	}
	339
	340	return changed;
	341	}
f9fbf1a3	342
db3eb96f AA	343	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	344	int pmdp_set_access_flags(struct vm_area_struct *vma,
	345	unsigned long address, pmd_t *pmdp,
	346	pmd_t entry, int dirty)
	347	{
	348	int changed = !pmd_same(*pmdp, entry);
	349
	350	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	351
	352	if (changed && dirty) {
	353	*pmdp = entry;
	354	pmd_update_defer(vma->vm_mm, address, pmdp);
5e4bf1a5 IM	355	/*
	356	* We had a write-protection fault here and changed the pmd
	357	* to to more permissive. No need to flush the TLB for that,
	358	* #PF is architecturally guaranteed to do that and in the
	359	* worst-case we'll generate a spurious fault.
	360	*/
db3eb96f AA	361	}
	362
	363	return changed;
	364	}
	365	#endif
	366
f9fbf1a3 JF	367	int ptep_test_and_clear_young(struct vm_area_struct *vma,
	368	unsigned long addr, pte_t *ptep)
	369	{
	370	int ret = 0;
	371
	372	if (pte_young(*ptep))
	373	ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
48e23957	374	(unsigned long *) &ptep->pte);
f9fbf1a3 JF	375
	376	if (ret)
	377	pte_update(vma->vm_mm, addr, ptep);
	378
	379	return ret;
	380	}
c20311e1	381
db3eb96f AA	382	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	383	int pmdp_test_and_clear_young(struct vm_area_struct *vma,
	384	unsigned long addr, pmd_t *pmdp)
	385	{
	386	int ret = 0;
	387
	388	if (pmd_young(*pmdp))
	389	ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
f2d6bfe9	390	(unsigned long *)pmdp);
db3eb96f AA	391
	392	if (ret)
	393	pmd_update(vma->vm_mm, addr, pmdp);
	394
	395	return ret;
	396	}
	397	#endif
	398
c20311e1 JF	399	int ptep_clear_flush_young(struct vm_area_struct *vma,
	400	unsigned long address, pte_t *ptep)
	401	{
	402	int young;
	403
	404	young = ptep_test_and_clear_young(vma, address, ptep);
	405	if (young)
	406	flush_tlb_page(vma, address);
	407
	408	return young;
	409	}
7c7e6e07	410
db3eb96f AA	411	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	412	int pmdp_clear_flush_young(struct vm_area_struct *vma,
	413	unsigned long address, pmd_t *pmdp)
	414	{
	415	int young;
	416
	417	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	418
	419	young = pmdp_test_and_clear_young(vma, address, pmdp);
	420	if (young)
	421	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	422
	423	return young;
	424	}
	425
	426	void pmdp_splitting_flush(struct vm_area_struct *vma,
	427	unsigned long address, pmd_t *pmdp)
	428	{
	429	int set;
	430	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	431	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
f2d6bfe9	432	(unsigned long *)pmdp);
db3eb96f AA	433	if (set) {
	434	pmd_update(vma->vm_mm, address, pmdp);
	435	/* need tlb flush only to serialize against gup-fast */
	436	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	437	}
	438	}
	439	#endif
	440
fd862dde GP	441	/**
	442	* reserve_top_address - reserves a hole in the top of kernel address space
	443	* @reserve - size of hole to reserve
	444	*
	445	* Can be used to relocate the fixmap area and poke a hole in the top
	446	* of kernel address space to make room for a hypervisor.
	447	*/
	448	void __init reserve_top_address(unsigned long reserve)
	449	{
	450	#ifdef CONFIG_X86_32
	451	BUG_ON(fixmaps_set > 0);
	452	printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
	453	(int)-reserve);
	454	__FIXADDR_TOP = -reserve - PAGE_SIZE;
fd862dde GP	455	#endif
	456	}
	457
7c7e6e07 JF	458	int fixmaps_set;
7c7e6e07 JF	459
aeaaa59c	460	void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
7c7e6e07 JF	461	{
	462	unsigned long address = __fix_to_virt(idx);
	463
	464	if (idx >= __end_of_fixed_addresses) {
	465	BUG();
	466	return;
	467	}
aeaaa59c	468	set_pte_vaddr(address, pte);
7c7e6e07 JF	469	fixmaps_set++;
7c7e6e07 JF	470	}
aeaaa59c	471
3b3809ac MH	472	void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
3b3809ac MH	473	pgprot_t flags)
aeaaa59c JF	474	{
	475	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
	476	}