[thirdparty/kernel/stable.git] / arch / sparc / mm / hugetlbpage.c

// SPDX-License-Identifier: GPL-2.0
/*
 * SPARC64 Huge TLB page support.
 *
 * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
 */

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/sysctl.h>

#include <asm/mman.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>

/* Slightly simplified from the non-hugepage variant because by
 * definition we don't have to worry about any page coloring stuff
 */

static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
							unsigned long addr,
							unsigned long len,
							unsigned long pgoff,
							unsigned long flags)
{
	struct hstate *h = hstate_file(filp);
	unsigned long task_size = TASK_SIZE;
	struct vm_unmapped_area_info info;

	if (test_thread_flag(TIF_32BIT))
		task_size = STACK_TOP32;

	info.flags = 0;
	info.length = len;
	info.low_limit = TASK_UNMAPPED_BASE;
	info.high_limit = min(task_size, VA_EXCLUDE_START);
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
		VM_BUG_ON(addr != -ENOMEM);
		info.low_limit = VA_EXCLUDE_END;
		info.high_limit = task_size;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

static unsigned long
hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
				  const unsigned long len,
				  const unsigned long pgoff,
				  const unsigned long flags)
{
	struct hstate *h = hstate_file(filp);
	struct mm_struct *mm = current->mm;
	unsigned long addr = addr0;
	struct vm_unmapped_area_info info;

	/* This should only ever run for 32-bit processes.  */
	BUG_ON(!test_thread_flag(TIF_32BIT));

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = mm->mmap_base;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
	if (addr & ~PAGE_MASK) {
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = STACK_TOP32;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long task_size = TASK_SIZE;

	if (test_thread_flag(TIF_32BIT))
		task_size = STACK_TOP32;

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (len > task_size)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		vma = find_vma(mm, addr);
		if (task_size - len >= addr &&
		    (!vma || addr + len <= vm_start_gap(vma)))
			return addr;
	}
	if (mm->get_unmapped_area == arch_get_unmapped_area)
		return hugetlb_get_unmapped_area_bottomup(file, addr, len,
				pgoff, flags);
	else
		return hugetlb_get_unmapped_area_topdown(file, addr, len,
				pgoff, flags);
}

static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
{
	return entry;
}

static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
{
	unsigned long hugepage_size = _PAGE_SZ4MB_4V;

	pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;

	switch (shift) {
	case HPAGE_16GB_SHIFT:
		hugepage_size = _PAGE_SZ16GB_4V;
		pte_val(entry) |= _PAGE_PUD_HUGE;
		break;
	case HPAGE_2GB_SHIFT:
		hugepage_size = _PAGE_SZ2GB_4V;
		pte_val(entry) |= _PAGE_PMD_HUGE;
		break;
	case HPAGE_256MB_SHIFT:
		hugepage_size = _PAGE_SZ256MB_4V;
		pte_val(entry) |= _PAGE_PMD_HUGE;
		break;
	case HPAGE_SHIFT:
		pte_val(entry) |= _PAGE_PMD_HUGE;
		break;
	case HPAGE_64K_SHIFT:
		hugepage_size = _PAGE_SZ64K_4V;
		break;
	default:
		WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
	}

	pte_val(entry) = pte_val(entry) | hugepage_size;
	return entry;
}

static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
{
	if (tlb_type == hypervisor)
		return sun4v_hugepage_shift_to_tte(entry, shift);
	else
		return sun4u_hugepage_shift_to_tte(entry, shift);
}

pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
{
	pte_t pte;

	entry = pte_mkhuge(entry);
	pte = hugepage_shift_to_tte(entry, shift);

#ifdef CONFIG_SPARC64
	/* If this vma has ADI enabled on it, turn on TTE.mcd
	 */
	if (flags & VM_SPARC_ADI)
		return pte_mkmcd(pte);
	else
		return pte_mknotmcd(pte);
#else
	return pte;
#endif
}

static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
{
	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
	unsigned int shift;

	switch (tte_szbits) {
	case _PAGE_SZ16GB_4V:
		shift = HPAGE_16GB_SHIFT;
		break;
	case _PAGE_SZ2GB_4V:
		shift = HPAGE_2GB_SHIFT;
		break;
	case _PAGE_SZ256MB_4V:
		shift = HPAGE_256MB_SHIFT;
		break;
	case _PAGE_SZ4MB_4V:
		shift = REAL_HPAGE_SHIFT;
		break;
	case _PAGE_SZ64K_4V:
		shift = HPAGE_64K_SHIFT;
		break;
	default:
		shift = PAGE_SHIFT;
		break;
	}
	return shift;
}

static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
{
	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
	unsigned int shift;

	switch (tte_szbits) {
	case _PAGE_SZ256MB_4U:
		shift = HPAGE_256MB_SHIFT;
		break;
	case _PAGE_SZ4MB_4U:
		shift = REAL_HPAGE_SHIFT;
		break;
	case _PAGE_SZ64K_4U:
		shift = HPAGE_64K_SHIFT;
		break;
	default:
		shift = PAGE_SHIFT;
		break;
	}
	return shift;
}

static unsigned long tte_to_shift(pte_t entry)
{
	if (tlb_type == hypervisor)
		return sun4v_huge_tte_to_shift(entry);

	return sun4u_huge_tte_to_shift(entry);
}

static unsigned int huge_tte_to_shift(pte_t entry)
{
	unsigned long shift = tte_to_shift(entry);

	if (shift == PAGE_SHIFT)
		WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
			  pte_val(entry));

	return shift;
}

static unsigned long huge_tte_to_size(pte_t pte)
{
	unsigned long size = 1UL << huge_tte_to_shift(pte);

	if (size == REAL_HPAGE_SIZE)
		size = HPAGE_SIZE;
	return size;
}

unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); }
unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); }
unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); }

pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
			unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	p4d = p4d_offset(pgd, addr);
	pud = pud_alloc(mm, p4d, addr);
	if (!pud)
		return NULL;
	if (sz >= PUD_SIZE)
		return (pte_t *)pud;
	pmd = pmd_alloc(mm, pud, addr);
	if (!pmd)
		return NULL;
	if (sz >= PMD_SIZE)
		return (pte_t *)pmd;
	return pte_alloc_huge(mm, pmd, addr);
}

pte_t *huge_pte_offset(struct mm_struct *mm,
		       unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	if (pgd_none(*pgd))
		return NULL;
	p4d = p4d_offset(pgd, addr);
	if (p4d_none(*p4d))
		return NULL;
	pud = pud_offset(p4d, addr);
	if (pud_none(*pud))
		return NULL;
	if (is_hugetlb_pud(*pud))
		return (pte_t *)pud;
	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return NULL;
	if (is_hugetlb_pmd(*pmd))
		return (pte_t *)pmd;
	return pte_offset_huge(pmd, addr);
}

void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t entry)
{
	unsigned int nptes, orig_shift, shift;
	unsigned long i, size;
	pte_t orig;

	size = huge_tte_to_size(entry);

	shift = PAGE_SHIFT;
	if (size >= PUD_SIZE)
		shift = PUD_SHIFT;
	else if (size >= PMD_SIZE)
		shift = PMD_SHIFT;
	else
		shift = PAGE_SHIFT;

	nptes = size >> shift;

	if (!pte_present(*ptep) && pte_present(entry))
		mm->context.hugetlb_pte_count += nptes;

	addr &= ~(size - 1);
	orig = *ptep;
	orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);

	for (i = 0; i < nptes; i++)
		ptep[i] = __pte(pte_val(entry) + (i << shift));

	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
	if (size == HPAGE_SIZE)
		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
				    orig_shift);
}

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t entry, unsigned long sz)
{
	__set_huge_pte_at(mm, addr, ptep, entry);
}

pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
			      pte_t *ptep)
{
	unsigned int i, nptes, orig_shift, shift;
	unsigned long size;
	pte_t entry;

	entry = *ptep;
	size = huge_tte_to_size(entry);

	shift = PAGE_SHIFT;
	if (size >= PUD_SIZE)
		shift = PUD_SHIFT;
	else if (size >= PMD_SIZE)
		shift = PMD_SHIFT;
	else
		shift = PAGE_SHIFT;

	nptes = size >> shift;
	orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);

	if (pte_present(entry))
		mm->context.hugetlb_pte_count -= nptes;

	addr &= ~(size - 1);
	for (i = 0; i < nptes; i++)
		ptep[i] = __pte(0UL);

	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
	if (size == HPAGE_SIZE)
		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
				    orig_shift);

	return entry;
}

int pmd_huge(pmd_t pmd)
{
	return !pmd_none(pmd) &&
		(pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID;
}

int pud_huge(pud_t pud)
{
	return !pud_none(pud) &&
		(pud_val(pud) & (_PAGE_VALID|_PAGE_PUD_HUGE)) != _PAGE_VALID;
}

static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
			   unsigned long addr)
{
	pgtable_t token = pmd_pgtable(*pmd);

	pmd_clear(pmd);
	pte_free_tlb(tlb, token, addr);
	mm_dec_nr_ptes(tlb->mm);
}

static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
				   unsigned long addr, unsigned long end,
				   unsigned long floor, unsigned long ceiling)
{
	pmd_t *pmd;
	unsigned long next;
	unsigned long start;

	start = addr;
	pmd = pmd_offset(pud, addr);
	do {
		next = pmd_addr_end(addr, end);
		if (pmd_none(*pmd))
			continue;
		if (is_hugetlb_pmd(*pmd))
			pmd_clear(pmd);
		else
			hugetlb_free_pte_range(tlb, pmd, addr);
	} while (pmd++, addr = next, addr != end);

	start &= PUD_MASK;
	if (start < floor)
		return;
	if (ceiling) {
		ceiling &= PUD_MASK;
		if (!ceiling)
			return;
	}
	if (end - 1 > ceiling - 1)
		return;

	pmd = pmd_offset(pud, start);
	pud_clear(pud);
	pmd_free_tlb(tlb, pmd, start);
	mm_dec_nr_pmds(tlb->mm);
}

static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
				   unsigned long addr, unsigned long end,
				   unsigned long floor, unsigned long ceiling)
{
	pud_t *pud;
	unsigned long next;
	unsigned long start;

	start = addr;
	pud = pud_offset(p4d, addr);
	do {
		next = pud_addr_end(addr, end);
		if (pud_none_or_clear_bad(pud))
			continue;
		if (is_hugetlb_pud(*pud))
			pud_clear(pud);
		else
			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
					       ceiling);
	} while (pud++, addr = next, addr != end);

	start &= PGDIR_MASK;
	if (start < floor)
		return;
	if (ceiling) {
		ceiling &= PGDIR_MASK;
		if (!ceiling)
			return;
	}
	if (end - 1 > ceiling - 1)
		return;

	pud = pud_offset(p4d, start);
	p4d_clear(p4d);
	pud_free_tlb(tlb, pud, start);
	mm_dec_nr_puds(tlb->mm);
}

void hugetlb_free_pgd_range(struct mmu_gather *tlb,
			    unsigned long addr, unsigned long end,
			    unsigned long floor, unsigned long ceiling)
{
	pgd_t *pgd;
	p4d_t *p4d;
	unsigned long next;

	addr &= PMD_MASK;
	if (addr < floor) {
		addr += PMD_SIZE;
		if (!addr)
			return;
	}
	if (ceiling) {
		ceiling &= PMD_MASK;
		if (!ceiling)
			return;
	}
	if (end - 1 > ceiling - 1)
		end -= PMD_SIZE;
	if (addr > end - 1)
		return;

	pgd = pgd_offset(tlb->mm, addr);
	p4d = p4d_offset(pgd, addr);
	do {
		next = p4d_addr_end(addr, end);
		if (p4d_none_or_clear_bad(p4d))
			continue;
		hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
	} while (p4d++, addr = next, addr != end);
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* SPARC64 Huge TLB page support.
	4	*
f6b83f07	5	* Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
1da177e4 LT	6	*/
1da177e4 LT	7
1da177e4 LT	8	#include <linux/fs.h>
1da177e4 LT	9	#include <linux/mm.h>
01042607	10	#include <linux/sched/mm.h>
1da177e4 LT	11	#include <linux/hugetlb.h>
1da177e4 LT	12	#include <linux/pagemap.h>
1da177e4 LT	13	#include <linux/sysctl.h>
	14
	15	#include <asm/mman.h>
	16	#include <asm/pgalloc.h>
	17	#include <asm/tlb.h>
	18	#include <asm/tlbflush.h>
	19	#include <asm/cacheflush.h>
	20	#include <asm/mmu_context.h>
	21
f6b83f07 DM	22	/* Slightly simplified from the non-hugepage variant because by
	23	* definition we don't have to worry about any page coloring stuff
	24	*/
f6b83f07 DM	25
	26	static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
	27	unsigned long addr,
	28	unsigned long len,
	29	unsigned long pgoff,
	30	unsigned long flags)
	31	{
c7d9f77d	32	struct hstate *h = hstate_file(filp);
f6b83f07	33	unsigned long task_size = TASK_SIZE;
2aea28b9	34	struct vm_unmapped_area_info info;
f6b83f07 DM	35
	36	if (test_thread_flag(TIF_32BIT))
	37	task_size = STACK_TOP32;
f6b83f07	38
2aea28b9 ML	39	info.flags = 0;
	40	info.length = len;
	41	info.low_limit = TASK_UNMAPPED_BASE;
	42	info.high_limit = min(task_size, VA_EXCLUDE_START);
c7d9f77d	43	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
2aea28b9 ML	44	info.align_offset = 0;
	45	addr = vm_unmapped_area(&info);
	46
	47	if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
	48	VM_BUG_ON(addr != -ENOMEM);
	49	info.low_limit = VA_EXCLUDE_END;
	50	info.high_limit = task_size;
	51	addr = vm_unmapped_area(&info);
f6b83f07 DM	52	}
f6b83f07 DM	53
2aea28b9	54	return addr;
f6b83f07 DM	55	}
	56
	57	static unsigned long
	58	hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
	59	const unsigned long len,
	60	const unsigned long pgoff,
	61	const unsigned long flags)
	62	{
c7d9f77d	63	struct hstate *h = hstate_file(filp);
f6b83f07 DM	64	struct mm_struct *mm = current->mm;
f6b83f07 DM	65	unsigned long addr = addr0;
2aea28b9	66	struct vm_unmapped_area_info info;
f6b83f07 DM	67
	68	/* This should only ever run for 32-bit processes. */
	69	BUG_ON(!test_thread_flag(TIF_32BIT));
	70
2aea28b9 ML	71	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	72	info.length = len;
	73	info.low_limit = PAGE_SIZE;
	74	info.high_limit = mm->mmap_base;
c7d9f77d	75	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
2aea28b9 ML	76	info.align_offset = 0;
2aea28b9 ML	77	addr = vm_unmapped_area(&info);
f6b83f07	78
f6b83f07 DM	79	/*
	80	* A failed mmap() very likely causes application failure,
	81	* so fall back to the bottom-up function here. This scenario
	82	* can happen with large stack limits and large mmap()
	83	* allocations.
	84	*/
2aea28b9 ML	85	if (addr & ~PAGE_MASK) {
	86	VM_BUG_ON(addr != -ENOMEM);
	87	info.flags = 0;
	88	info.low_limit = TASK_UNMAPPED_BASE;
	89	info.high_limit = STACK_TOP32;
	90	addr = vm_unmapped_area(&info);
	91	}
f6b83f07 DM	92
	93	return addr;
	94	}
	95
	96	unsigned long
	97	hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
	98	unsigned long len, unsigned long pgoff, unsigned long flags)
	99	{
c7d9f77d	100	struct hstate *h = hstate_file(file);
f6b83f07 DM	101	struct mm_struct *mm = current->mm;
	102	struct vm_area_struct *vma;
	103	unsigned long task_size = TASK_SIZE;
	104
	105	if (test_thread_flag(TIF_32BIT))
	106	task_size = STACK_TOP32;
	107
c7d9f77d	108	if (len & ~huge_page_mask(h))
f6b83f07 DM	109	return -EINVAL;
	110	if (len > task_size)
	111	return -ENOMEM;
	112
ac35ee48	113	if (flags & MAP_FIXED) {
a5516438	114	if (prepare_hugepage_range(file, addr, len))
ac35ee48 BH	115	return -EINVAL;
	116	return addr;
	117	}
	118
f6b83f07	119	if (addr) {
c7d9f77d	120	addr = ALIGN(addr, huge_page_size(h));
f6b83f07 DM	121	vma = find_vma(mm, addr);
f6b83f07 DM	122	if (task_size - len >= addr &&
1be7107f	123	(!vma \|\| addr + len <= vm_start_gap(vma)))
f6b83f07 DM	124	return addr;
	125	}
	126	if (mm->get_unmapped_area == arch_get_unmapped_area)
	127	return hugetlb_get_unmapped_area_bottomup(file, addr, len,
	128	pgoff, flags);
	129	else
	130	return hugetlb_get_unmapped_area_topdown(file, addr, len,
	131	pgoff, flags);
	132	}
	133
c7d9f77d NG	134	static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
	135	{
	136	return entry;
	137	}
	138
	139	static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
	140	{
	141	unsigned long hugepage_size = _PAGE_SZ4MB_4V;
	142
	143	pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
	144
	145	switch (shift) {
df7b2155 NG	146	case HPAGE_16GB_SHIFT:
	147	hugepage_size = _PAGE_SZ16GB_4V;
	148	pte_val(entry) \|= _PAGE_PUD_HUGE;
	149	break;
85b1da7c NG	150	case HPAGE_2GB_SHIFT:
	151	hugepage_size = _PAGE_SZ2GB_4V;
	152	pte_val(entry) \|= _PAGE_PMD_HUGE;
	153	break;
c7d9f77d NG	154	case HPAGE_256MB_SHIFT:
	155	hugepage_size = _PAGE_SZ256MB_4V;
	156	pte_val(entry) \|= _PAGE_PMD_HUGE;
	157	break;
	158	case HPAGE_SHIFT:
	159	pte_val(entry) \|= _PAGE_PMD_HUGE;
	160	break;
dcd1912d NG	161	case HPAGE_64K_SHIFT:
	162	hugepage_size = _PAGE_SZ64K_4V;
	163	break;
c7d9f77d NG	164	default:
	165	WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
	166	}
	167
	168	pte_val(entry) = pte_val(entry) \| hugepage_size;
	169	return entry;
	170	}
	171
	172	static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
	173	{
	174	if (tlb_type == hypervisor)
	175	return sun4v_hugepage_shift_to_tte(entry, shift);
	176	else
	177	return sun4u_hugepage_shift_to_tte(entry, shift);
	178	}
	179
79c1c594	180	pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
c7d9f77d	181	{
74a04967	182	pte_t pte;
c7d9f77d	183
16785bd7	184	entry = pte_mkhuge(entry);
74a04967 KA	185	pte = hugepage_shift_to_tte(entry, shift);
	186
	187	#ifdef CONFIG_SPARC64
	188	/* If this vma has ADI enabled on it, turn on TTE.mcd
	189	*/
79c1c594	190	if (flags & VM_SPARC_ADI)
74a04967 KA	191	return pte_mkmcd(pte);
	192	else
	193	return pte_mknotmcd(pte);
	194	#else
	195	return pte;
	196	#endif
c7d9f77d NG	197	}
	198
	199	static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
	200	{
	201	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
	202	unsigned int shift;
	203
	204	switch (tte_szbits) {
df7b2155 NG	205	case _PAGE_SZ16GB_4V:
	206	shift = HPAGE_16GB_SHIFT;
	207	break;
85b1da7c NG	208	case _PAGE_SZ2GB_4V:
	209	shift = HPAGE_2GB_SHIFT;
	210	break;
c7d9f77d NG	211	case _PAGE_SZ256MB_4V:
	212	shift = HPAGE_256MB_SHIFT;
	213	break;
	214	case _PAGE_SZ4MB_4V:
	215	shift = REAL_HPAGE_SHIFT;
	216	break;
dcd1912d NG	217	case _PAGE_SZ64K_4V:
	218	shift = HPAGE_64K_SHIFT;
	219	break;
c7d9f77d NG	220	default:
	221	shift = PAGE_SHIFT;
	222	break;
	223	}
	224	return shift;
	225	}
	226
	227	static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
	228	{
	229	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
	230	unsigned int shift;
	231
	232	switch (tte_szbits) {
	233	case _PAGE_SZ256MB_4U:
	234	shift = HPAGE_256MB_SHIFT;
	235	break;
	236	case _PAGE_SZ4MB_4U:
	237	shift = REAL_HPAGE_SHIFT;
	238	break;
dcd1912d NG	239	case _PAGE_SZ64K_4U:
	240	shift = HPAGE_64K_SHIFT;
	241	break;
c7d9f77d NG	242	default:
	243	shift = PAGE_SHIFT;
	244	break;
	245	}
	246	return shift;
	247	}
	248
e6e4f42e	249	static unsigned long tte_to_shift(pte_t entry)
c7d9f77d	250	{
c7d9f77d	251	if (tlb_type == hypervisor)
e6e4f42e PZ	252	return sun4v_huge_tte_to_shift(entry);
	253
	254	return sun4u_huge_tte_to_shift(entry);
	255	}
	256
	257	static unsigned int huge_tte_to_shift(pte_t entry)
	258	{
	259	unsigned long shift = tte_to_shift(entry);
c7d9f77d NG	260
	261	if (shift == PAGE_SHIFT)
	262	WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
	263	pte_val(entry));
	264
	265	return shift;
	266	}
	267
	268	static unsigned long huge_tte_to_size(pte_t pte)
	269	{
	270	unsigned long size = 1UL << huge_tte_to_shift(pte);
	271
	272	if (size == REAL_HPAGE_SIZE)
	273	size = HPAGE_SIZE;
	274	return size;
	275	}
	276
e6e4f42e PZ	277	unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift((pte_t )&pud); }
	278	unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift((pte_t )&pmd); }
	279	unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); }
	280
aec44e0f	281	pte_t huge_pte_alloc(struct mm_struct mm, struct vm_area_struct *vma,
a5516438	282	unsigned long addr, unsigned long sz)
1da177e4 LT	283	{
1da177e4 LT	284	pgd_t *pgd;
5637bc50	285	p4d_t *p4d;
1da177e4	286	pud_t *pud;
dcd1912d	287	pmd_t *pmd;
1da177e4 LT	288
1da177e4 LT	289	pgd = pgd_offset(mm, addr);
5637bc50 MR	290	p4d = p4d_offset(pgd, addr);
5637bc50 MR	291	pud = pud_alloc(mm, p4d, addr);
df7b2155 NG	292	if (!pud)
df7b2155 NG	293	return NULL;
df7b2155	294	if (sz >= PUD_SIZE)
4dbe87d5 NG	295	return (pte_t *)pud;
	296	pmd = pmd_alloc(mm, pud, addr);
	297	if (!pmd)
	298	return NULL;
	299	if (sz >= PMD_SIZE)
	300	return (pte_t *)pmd;
c65d09fd	301	return pte_alloc_huge(mm, pmd, addr);
1da177e4 LT	302	}
1da177e4 LT	303
7868a208 PA	304	pte_t huge_pte_offset(struct mm_struct mm,
7868a208 PA	305	unsigned long addr, unsigned long sz)
1da177e4 LT	306	{
1da177e4 LT	307	pgd_t *pgd;
5637bc50	308	p4d_t *p4d;
1da177e4	309	pud_t *pud;
dcd1912d	310	pmd_t *pmd;
1da177e4 LT	311
1da177e4 LT	312	pgd = pgd_offset(mm, addr);
4dbe87d5 NG	313	if (pgd_none(*pgd))
4dbe87d5 NG	314	return NULL;
5637bc50 MR	315	p4d = p4d_offset(pgd, addr);
	316	if (p4d_none(*p4d))
	317	return NULL;
	318	pud = pud_offset(p4d, addr);
4dbe87d5 NG	319	if (pud_none(*pud))
	320	return NULL;
	321	if (is_hugetlb_pud(*pud))
	322	return (pte_t *)pud;
	323	pmd = pmd_offset(pud, addr);
	324	if (pmd_none(*pmd))
	325	return NULL;
	326	if (is_hugetlb_pmd(*pmd))
	327	return (pte_t *)pmd;
c65d09fd	328	return pte_offset_huge(pmd, addr);
1da177e4 LT	329	}
1da177e4 LT	330
935d4f0c	331	void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
63551ae0	332	pte_t *ptep, pte_t entry)
1da177e4	333	{
df7b2155 NG	334	unsigned int nptes, orig_shift, shift;
df7b2155 NG	335	unsigned long i, size;
7bc3777c	336	pte_t orig;
63551ae0	337
c7d9f77d	338	size = huge_tte_to_size(entry);
df7b2155 NG	339
	340	shift = PAGE_SHIFT;
	341	if (size >= PUD_SIZE)
	342	shift = PUD_SHIFT;
	343	else if (size >= PMD_SIZE)
	344	shift = PMD_SHIFT;
	345	else
	346	shift = PAGE_SHIFT;
	347
dcd1912d	348	nptes = size >> shift;
c7d9f77d	349
dcc1e8dd	350	if (!pte_present(*ptep) && pte_present(entry))
c7d9f77d	351	mm->context.hugetlb_pte_count += nptes;
dcc1e8dd	352
c7d9f77d	353	addr &= ~(size - 1);
7bc3777c	354	orig = *ptep;
ac65e282	355	orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
24e49ee3	356
c7d9f77d	357	for (i = 0; i < nptes; i++)
dcd1912d	358	ptep[i] = __pte(pte_val(entry) + (i << shift));
c7d9f77d	359
dcd1912d	360	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
c7d9f77d NG	361	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
	362	if (size == HPAGE_SIZE)
	363	maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
dcd1912d	364	orig_shift);
63551ae0	365	}
1da177e4	366
935d4f0c RR	367	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
	368	pte_t *ptep, pte_t entry, unsigned long sz)
	369	{
	370	__set_huge_pte_at(mm, addr, ptep, entry);
	371	}
	372
63551ae0 DG	373	pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
	374	pte_t *ptep)
	375	{
df7b2155	376	unsigned int i, nptes, orig_shift, shift;
c7d9f77d	377	unsigned long size;
63551ae0	378	pte_t entry;
1da177e4	379
63551ae0	380	entry = *ptep;
c7d9f77d	381	size = huge_tte_to_size(entry);
df7b2155 NG	382
	383	shift = PAGE_SHIFT;
	384	if (size >= PUD_SIZE)
	385	shift = PUD_SHIFT;
	386	else if (size >= PMD_SIZE)
	387	shift = PMD_SHIFT;
dcd1912d	388	else
df7b2155	389	shift = PAGE_SHIFT;
dcd1912d	390
df7b2155 NG	391	nptes = size >> shift;
df7b2155 NG	392	orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
c7d9f77d	393
dcc1e8dd	394	if (pte_present(entry))
c7d9f77d	395	mm->context.hugetlb_pte_count -= nptes;
1da177e4	396
c7d9f77d NG	397	addr &= ~(size - 1);
	398	for (i = 0; i < nptes; i++)
	399	ptep[i] = __pte(0UL);
63551ae0	400
df7b2155	401	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
c7d9f77d NG	402	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
	403	if (size == HPAGE_SIZE)
	404	maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
df7b2155	405	orig_shift);
24e49ee3	406
63551ae0	407	return entry;
1da177e4 LT	408	}
1da177e4 LT	409
1da177e4 LT	410	int pmd_huge(pmd_t pmd)
1da177e4 LT	411	{
7bc3777c NG	412	return !pmd_none(pmd) &&
7bc3777c NG	413	(pmd_val(pmd) & (_PAGE_VALID\|_PAGE_PMD_HUGE)) != _PAGE_VALID;
1da177e4 LT	414	}
1da177e4 LT	415
ceb86879 AK	416	int pud_huge(pud_t pud)
ceb86879 AK	417	{
df7b2155 NG	418	return !pud_none(pud) &&
df7b2155 NG	419	(pud_val(pud) & (_PAGE_VALID\|_PAGE_PUD_HUGE)) != _PAGE_VALID;
ceb86879	420	}
7bc3777c NG	421
	422	static void hugetlb_free_pte_range(struct mmu_gather tlb, pmd_t pmd,
	423	unsigned long addr)
	424	{
	425	pgtable_t token = pmd_pgtable(*pmd);
	426
	427	pmd_clear(pmd);
	428	pte_free_tlb(tlb, token, addr);
c4812909	429	mm_dec_nr_ptes(tlb->mm);
7bc3777c NG	430	}
	431
	432	static void hugetlb_free_pmd_range(struct mmu_gather tlb, pud_t pud,
	433	unsigned long addr, unsigned long end,
	434	unsigned long floor, unsigned long ceiling)
	435	{
	436	pmd_t *pmd;
	437	unsigned long next;
	438	unsigned long start;
	439
	440	start = addr;
	441	pmd = pmd_offset(pud, addr);
	442	do {
	443	next = pmd_addr_end(addr, end);
	444	if (pmd_none(*pmd))
	445	continue;
	446	if (is_hugetlb_pmd(*pmd))
	447	pmd_clear(pmd);
	448	else
	449	hugetlb_free_pte_range(tlb, pmd, addr);
	450	} while (pmd++, addr = next, addr != end);
	451
	452	start &= PUD_MASK;
	453	if (start < floor)
	454	return;
	455	if (ceiling) {
	456	ceiling &= PUD_MASK;
	457	if (!ceiling)
	458	return;
	459	}
	460	if (end - 1 > ceiling - 1)
	461	return;
	462
	463	pmd = pmd_offset(pud, start);
	464	pud_clear(pud);
	465	pmd_free_tlb(tlb, pmd, start);
	466	mm_dec_nr_pmds(tlb->mm);
	467	}
	468
5637bc50	469	static void hugetlb_free_pud_range(struct mmu_gather tlb, p4d_t p4d,
7bc3777c NG	470	unsigned long addr, unsigned long end,
	471	unsigned long floor, unsigned long ceiling)
	472	{
	473	pud_t *pud;
	474	unsigned long next;
	475	unsigned long start;
	476
	477	start = addr;
5637bc50	478	pud = pud_offset(p4d, addr);
7bc3777c NG	479	do {
	480	next = pud_addr_end(addr, end);
	481	if (pud_none_or_clear_bad(pud))
	482	continue;
df7b2155 NG	483	if (is_hugetlb_pud(*pud))
	484	pud_clear(pud);
	485	else
	486	hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
	487	ceiling);
7bc3777c NG	488	} while (pud++, addr = next, addr != end);
	489
	490	start &= PGDIR_MASK;
	491	if (start < floor)
	492	return;
	493	if (ceiling) {
	494	ceiling &= PGDIR_MASK;
	495	if (!ceiling)
	496	return;
	497	}
	498	if (end - 1 > ceiling - 1)
	499	return;
	500
5637bc50 MR	501	pud = pud_offset(p4d, start);
5637bc50 MR	502	p4d_clear(p4d);
7bc3777c	503	pud_free_tlb(tlb, pud, start);
b4e98d9a	504	mm_dec_nr_puds(tlb->mm);
7bc3777c NG	505	}
	506
	507	void hugetlb_free_pgd_range(struct mmu_gather *tlb,
	508	unsigned long addr, unsigned long end,
	509	unsigned long floor, unsigned long ceiling)
	510	{
	511	pgd_t *pgd;
5637bc50	512	p4d_t *p4d;
7bc3777c NG	513	unsigned long next;
7bc3777c NG	514
544f8f93 NG	515	addr &= PMD_MASK;
	516	if (addr < floor) {
	517	addr += PMD_SIZE;
	518	if (!addr)
	519	return;
	520	}
	521	if (ceiling) {
	522	ceiling &= PMD_MASK;
	523	if (!ceiling)
	524	return;
	525	}
	526	if (end - 1 > ceiling - 1)
	527	end -= PMD_SIZE;
	528	if (addr > end - 1)
	529	return;
	530
7bc3777c	531	pgd = pgd_offset(tlb->mm, addr);
5637bc50	532	p4d = p4d_offset(pgd, addr);
7bc3777c	533	do {
5637bc50 MR	534	next = p4d_addr_end(addr, end);
5637bc50 MR	535	if (p4d_none_or_clear_bad(p4d))
7bc3777c	536	continue;
5637bc50 MR	537	hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
5637bc50 MR	538	} while (p4d++, addr = next, addr != end);
7bc3777c	539	}