[thirdparty/u-boot.git] / arch / arm / cpu / armv8 / cache_v8.c

/*
 * (C) Copyright 2013
 * David Feng <fenghua@phytium.com.cn>
 *
 * (C) Copyright 2016
 * Alexander Graf <agraf@suse.de>
 *
 * SPDX-License-Identifier:	GPL-2.0+
 */

#include <common.h>
#include <asm/system.h>
#include <asm/armv8/mmu.h>

DECLARE_GLOBAL_DATA_PTR;

#ifndef CONFIG_SYS_DCACHE_OFF

/*
 *  With 4k page granule, a virtual address is split into 4 lookup parts
 *  spanning 9 bits each:
 *
 *    _______________________________________________
 *   |       |       |       |       |       |       |
 *   |   0   |  Lv0  |  Lv1  |  Lv2  |  Lv3  |  off  |
 *   |_______|_______|_______|_______|_______|_______|
 *     63-48   47-39   38-30   29-21   20-12   11-00
 *
 *             mask        page size
 *
 *    Lv0: FF8000000000       --
 *    Lv1:   7FC0000000       1G
 *    Lv2:     3FE00000       2M
 *    Lv3:       1FF000       4K
 *    off:          FFF
 */

u64 get_tcr(int el, u64 *pips, u64 *pva_bits)
{
	u64 max_addr = 0;
	u64 ips, va_bits;
	u64 tcr;
	int i;

	/* Find the largest address we need to support */
	for (i = 0; mem_map[i].size || mem_map[i].attrs; i++)
		max_addr = max(max_addr, mem_map[i].virt + mem_map[i].size);

	/* Calculate the maximum physical (and thus virtual) address */
	if (max_addr > (1ULL << 44)) {
		ips = 5;
		va_bits = 48;
	} else  if (max_addr > (1ULL << 42)) {
		ips = 4;
		va_bits = 44;
	} else  if (max_addr > (1ULL << 40)) {
		ips = 3;
		va_bits = 42;
	} else  if (max_addr > (1ULL << 36)) {
		ips = 2;
		va_bits = 40;
	} else  if (max_addr > (1ULL << 32)) {
		ips = 1;
		va_bits = 36;
	} else {
		ips = 0;
		va_bits = 32;
	}

	if (el == 1) {
		tcr = TCR_EL1_RSVD | (ips << 32) | TCR_EPD1_DISABLE;
	} else if (el == 2) {
		tcr = TCR_EL2_RSVD | (ips << 16);
	} else {
		tcr = TCR_EL3_RSVD | (ips << 16);
	}

	/* PTWs cacheable, inner/outer WBWA and inner shareable */
	tcr |= TCR_TG0_4K | TCR_SHARED_INNER | TCR_ORGN_WBWA | TCR_IRGN_WBWA;
	tcr |= TCR_T0SZ(va_bits);

	if (pips)
		*pips = ips;
	if (pva_bits)
		*pva_bits = va_bits;

	return tcr;
}

#define MAX_PTE_ENTRIES 512

static int pte_type(u64 *pte)
{
	return *pte & PTE_TYPE_MASK;
}

/* Returns the LSB number for a PTE on level <level> */
static int level2shift(int level)
{
	/* Page is 12 bits wide, every level translates 9 bits */
	return (12 + 9 * (3 - level));
}

static u64 *find_pte(u64 addr, int level)
{
	int start_level = 0;
	u64 *pte;
	u64 idx;
	u64 va_bits;
	int i;

	debug("addr=%llx level=%d\n", addr, level);

	get_tcr(0, NULL, &va_bits);
	if (va_bits < 39)
		start_level = 1;

	if (level < start_level)
		return NULL;

	/* Walk through all page table levels to find our PTE */
	pte = (u64*)gd->arch.tlb_addr;
	for (i = start_level; i < 4; i++) {
		idx = (addr >> level2shift(i)) & 0x1FF;
		pte += idx;
		debug("idx=%llx PTE %p at level %d: %llx\n", idx, pte, i, *pte);

		/* Found it */
		if (i == level)
			return pte;
		/* PTE is no table (either invalid or block), can't traverse */
		if (pte_type(pte) != PTE_TYPE_TABLE)
			return NULL;
		/* Off to the next level */
		pte = (u64*)(*pte & 0x0000fffffffff000ULL);
	}

	/* Should never reach here */
	return NULL;
}

/* Returns and creates a new full table (512 entries) */
static u64 *create_table(void)
{
	u64 *new_table = (u64*)gd->arch.tlb_fillptr;
	u64 pt_len = MAX_PTE_ENTRIES * sizeof(u64);

	/* Allocate MAX_PTE_ENTRIES pte entries */
	gd->arch.tlb_fillptr += pt_len;

	if (gd->arch.tlb_fillptr - gd->arch.tlb_addr > gd->arch.tlb_size)
		panic("Insufficient RAM for page table: 0x%lx > 0x%lx. "
		      "Please increase the size in get_page_table_size()",
			gd->arch.tlb_fillptr - gd->arch.tlb_addr,
			gd->arch.tlb_size);

	/* Mark all entries as invalid */
	memset(new_table, 0, pt_len);

	return new_table;
}

static void set_pte_table(u64 *pte, u64 *table)
{
	/* Point *pte to the new table */
	debug("Setting %p to addr=%p\n", pte, table);
	*pte = PTE_TYPE_TABLE | (ulong)table;
}

/* Splits a block PTE into table with subpages spanning the old block */
static void split_block(u64 *pte, int level)
{
	u64 old_pte = *pte;
	u64 *new_table;
	u64 i = 0;
	/* level describes the parent level, we need the child ones */
	int levelshift = level2shift(level + 1);

	if (pte_type(pte) != PTE_TYPE_BLOCK)
		panic("PTE %p (%llx) is not a block. Some driver code wants to "
		      "modify dcache settings for an range not covered in "
		      "mem_map.", pte, old_pte);

	new_table = create_table();
	debug("Splitting pte %p (%llx) into %p\n", pte, old_pte, new_table);

	for (i = 0; i < MAX_PTE_ENTRIES; i++) {
		new_table[i] = old_pte | (i << levelshift);

		/* Level 3 block PTEs have the table type */
		if ((level + 1) == 3)
			new_table[i] |= PTE_TYPE_TABLE;

		debug("Setting new_table[%lld] = %llx\n", i, new_table[i]);
	}

	/* Set the new table into effect */
	set_pte_table(pte, new_table);
}

/* Add one mm_region map entry to the page tables */
static void add_map(struct mm_region *map)
{
	u64 *pte;
	u64 virt = map->virt;
	u64 phys = map->phys;
	u64 size = map->size;
	u64 attrs = map->attrs | PTE_TYPE_BLOCK | PTE_BLOCK_AF;
	u64 blocksize;
	int level;
	u64 *new_table;

	while (size) {
		pte = find_pte(virt, 0);
		if (pte && (pte_type(pte) == PTE_TYPE_FAULT)) {
			debug("Creating table for virt 0x%llx\n", virt);
			new_table = create_table();
			set_pte_table(pte, new_table);
		}

		for (level = 1; level < 4; level++) {
			pte = find_pte(virt, level);
			if (!pte)
				panic("pte not found\n");

			blocksize = 1ULL << level2shift(level);
			debug("Checking if pte fits for virt=%llx size=%llx blocksize=%llx\n",
			      virt, size, blocksize);
			if (size >= blocksize && !(virt & (blocksize - 1))) {
				/* Page fits, create block PTE */
				debug("Setting PTE %p to block virt=%llx\n",
				      pte, virt);
				*pte = phys | attrs;
				virt += blocksize;
				phys += blocksize;
				size -= blocksize;
				break;
			} else if (pte_type(pte) == PTE_TYPE_FAULT) {
				/* Page doesn't fit, create subpages */
				debug("Creating subtable for virt 0x%llx blksize=%llx\n",
				      virt, blocksize);
				new_table = create_table();
				set_pte_table(pte, new_table);
			} else if (pte_type(pte) == PTE_TYPE_BLOCK) {
				debug("Split block into subtable for virt 0x%llx blksize=0x%llx\n",
				      virt, blocksize);
				split_block(pte, level);
			}
		}
	}
}

enum pte_type {
	PTE_INVAL,
	PTE_BLOCK,
	PTE_LEVEL,
};

/*
 * This is a recursively called function to count the number of
 * page tables we need to cover a particular PTE range. If you
 * call this with level = -1 you basically get the full 48 bit
 * coverage.
 */
static int count_required_pts(u64 addr, int level, u64 maxaddr)
{
	int levelshift = level2shift(level);
	u64 levelsize = 1ULL << levelshift;
	u64 levelmask = levelsize - 1;
	u64 levelend = addr + levelsize;
	int r = 0;
	int i;
	enum pte_type pte_type = PTE_INVAL;

	for (i = 0; mem_map[i].size || mem_map[i].attrs; i++) {
		struct mm_region *map = &mem_map[i];
		u64 start = map->virt;
		u64 end = start + map->size;

		/* Check if the PTE would overlap with the map */
		if (max(addr, start) <= min(levelend, end)) {
			start = max(addr, start);
			end = min(levelend, end);

			/* We need a sub-pt for this level */
			if ((start & levelmask) || (end & levelmask)) {
				pte_type = PTE_LEVEL;
				break;
			}

			/* Lv0 can not do block PTEs, so do levels here too */
			if (level <= 0) {
				pte_type = PTE_LEVEL;
				break;
			}

			/* PTE is active, but fits into a block */
			pte_type = PTE_BLOCK;
		}
	}

	/*
	 * Block PTEs at this level are already covered by the parent page
	 * table, so we only need to count sub page tables.
	 */
	if (pte_type == PTE_LEVEL) {
		int sublevel = level + 1;
		u64 sublevelsize = 1ULL << level2shift(sublevel);

		/* Account for the new sub page table ... */
		r = 1;

		/* ... and for all child page tables that one might have */
		for (i = 0; i < MAX_PTE_ENTRIES; i++) {
			r += count_required_pts(addr, sublevel, maxaddr);
			addr += sublevelsize;

			if (addr >= maxaddr) {
				/*
				 * We reached the end of address space, no need
				 * to look any further.
				 */
				break;
			}
		}
	}

	return r;
}

/* Returns the estimated required size of all page tables */
__weak u64 get_page_table_size(void)
{
	u64 one_pt = MAX_PTE_ENTRIES * sizeof(u64);
	u64 size = 0;
	u64 va_bits;
	int start_level = 0;

	get_tcr(0, NULL, &va_bits);
	if (va_bits < 39)
		start_level = 1;

	/* Account for all page tables we would need to cover our memory map */
	size = one_pt * count_required_pts(0, start_level - 1, 1ULL << va_bits);

	/*
	 * We need to duplicate our page table once to have an emergency pt to
	 * resort to when splitting page tables later on
	 */
	size *= 2;

	/*
	 * We may need to split page tables later on if dcache settings change,
	 * so reserve up to 4 (random pick) page tables for that.
	 */
	size += one_pt * 4;

	return size;
}

void setup_pgtables(void)
{
	int i;

	if (!gd->arch.tlb_fillptr || !gd->arch.tlb_addr)
		panic("Page table pointer not setup.");

	/*
	 * Allocate the first level we're on with invalidate entries.
	 * If the starting level is 0 (va_bits >= 39), then this is our
	 * Lv0 page table, otherwise it's the entry Lv1 page table.
	 */
	create_table();

	/* Now add all MMU table entries one after another to the table */
	for (i = 0; mem_map[i].size || mem_map[i].attrs; i++)
		add_map(&mem_map[i]);
}

static void setup_all_pgtables(void)
{
	u64 tlb_addr = gd->arch.tlb_addr;
	u64 tlb_size = gd->arch.tlb_size;

	/* Reset the fill ptr */
	gd->arch.tlb_fillptr = tlb_addr;

	/* Create normal system page tables */
	setup_pgtables();

	/* Create emergency page tables */
	gd->arch.tlb_size -= (uintptr_t)gd->arch.tlb_fillptr -
			     (uintptr_t)gd->arch.tlb_addr;
	gd->arch.tlb_addr = gd->arch.tlb_fillptr;
	setup_pgtables();
	gd->arch.tlb_emerg = gd->arch.tlb_addr;
	gd->arch.tlb_addr = tlb_addr;
	gd->arch.tlb_size = tlb_size;
}

/* to activate the MMU we need to set up virtual memory */
__weak void mmu_setup(void)
{
	int el;

	/* Set up page tables only once */
	if (!gd->arch.tlb_fillptr)
		setup_all_pgtables();

	el = current_el();
	set_ttbr_tcr_mair(el, gd->arch.tlb_addr, get_tcr(el, NULL, NULL),
			  MEMORY_ATTRIBUTES);

	/* enable the mmu */
	set_sctlr(get_sctlr() | CR_M);
}

/*
 * Performs a invalidation of the entire data cache at all levels
 */
void invalidate_dcache_all(void)
{
	__asm_invalidate_dcache_all();
	__asm_invalidate_l3_dcache();
}

/*
 * Performs a clean & invalidation of the entire data cache at all levels.
 * This function needs to be inline to avoid using stack.
 * __asm_flush_l3_dcache return status of timeout
 */
inline void flush_dcache_all(void)
{
	int ret;

	__asm_flush_dcache_all();
	ret = __asm_flush_l3_dcache();
	if (ret)
		debug("flushing dcache returns 0x%x\n", ret);
	else
		debug("flushing dcache successfully.\n");
}

/*
 * Invalidates range in all levels of D-cache/unified cache
 */
void invalidate_dcache_range(unsigned long start, unsigned long stop)
{
	__asm_flush_dcache_range(start, stop);
}

/*
 * Flush range(clean & invalidate) from all levels of D-cache/unified cache
 */
void flush_dcache_range(unsigned long start, unsigned long stop)
{
	__asm_flush_dcache_range(start, stop);
}

void dcache_enable(void)
{
	/* The data cache is not active unless the mmu is enabled */
	if (!(get_sctlr() & CR_M)) {
		invalidate_dcache_all();
		__asm_invalidate_tlb_all();
		mmu_setup();
	}

	set_sctlr(get_sctlr() | CR_C);
}

void dcache_disable(void)
{
	uint32_t sctlr;

	sctlr = get_sctlr();

	/* if cache isn't enabled no need to disable */
	if (!(sctlr & CR_C))
		return;

	set_sctlr(sctlr & ~(CR_C|CR_M));

	flush_dcache_all();
	__asm_invalidate_tlb_all();
}

int dcache_status(void)
{
	return (get_sctlr() & CR_C) != 0;
}

u64 *__weak arch_get_page_table(void) {
	puts("No page table offset defined\n");

	return NULL;
}

static bool is_aligned(u64 addr, u64 size, u64 align)
{
	return !(addr & (align - 1)) && !(size & (align - 1));
}

static u64 set_one_region(u64 start, u64 size, u64 attrs, int level)
{
	int levelshift = level2shift(level);
	u64 levelsize = 1ULL << levelshift;
	u64 *pte = find_pte(start, level);

	/* Can we can just modify the current level block PTE? */
	if (is_aligned(start, size, levelsize)) {
		*pte &= ~PMD_ATTRINDX_MASK;
		*pte |= attrs;
		debug("Set attrs=%llx pte=%p level=%d\n", attrs, pte, level);

		return levelsize;
	}

	/* Unaligned or doesn't fit, maybe split block into table */
	debug("addr=%llx level=%d pte=%p (%llx)\n", start, level, pte, *pte);

	/* Maybe we need to split the block into a table */
	if (pte_type(pte) == PTE_TYPE_BLOCK)
		split_block(pte, level);

	/* And then double-check it became a table or already is one */
	if (pte_type(pte) != PTE_TYPE_TABLE)
		panic("PTE %p (%llx) for addr=%llx should be a table",
		      pte, *pte, start);

	/* Roll on to the next page table level */
	return 0;
}

void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size,
				     enum dcache_option option)
{
	u64 attrs = PMD_ATTRINDX(option);
	u64 real_start = start;
	u64 real_size = size;

	debug("start=%lx size=%lx\n", (ulong)start, (ulong)size);

	if (!gd->arch.tlb_emerg)
		panic("Emergency page table not setup.");

	/*
	 * We can not modify page tables that we're currently running on,
	 * so we first need to switch to the "emergency" page tables where
	 * we can safely modify our primary page tables and then switch back
	 */
	__asm_switch_ttbr(gd->arch.tlb_emerg);

	/*
	 * Loop through the address range until we find a page granule that fits
	 * our alignment constraints, then set it to the new cache attributes
	 */
	while (size > 0) {
		int level;
		u64 r;

		for (level = 1; level < 4; level++) {
			r = set_one_region(start, size, attrs, level);
			if (r) {
				/* PTE successfully replaced */
				size -= r;
				start += r;
				break;
			}
		}

	}

	/* We're done modifying page tables, switch back to our primary ones */
	__asm_switch_ttbr(gd->arch.tlb_addr);

	/*
	 * Make sure there's nothing stale in dcache for a region that might
	 * have caches off now
	 */
	flush_dcache_range(real_start, real_start + real_size);
}

#else	/* CONFIG_SYS_DCACHE_OFF */

/*
 * For SPL builds, we may want to not have dcache enabled. Any real U-Boot
 * running however really wants to have dcache and the MMU active. Check that
 * everything is sane and give the developer a hint if it isn't.
 */
#ifndef CONFIG_SPL_BUILD
#error Please describe your MMU layout in CONFIG_SYS_MEM_MAP and enable dcache.
#endif

void invalidate_dcache_all(void)
{
}

void flush_dcache_all(void)
{
}

void dcache_enable(void)
{
}

void dcache_disable(void)
{
}

int dcache_status(void)
{
	return 0;
}

void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size,
				     enum dcache_option option)
{
}

#endif	/* CONFIG_SYS_DCACHE_OFF */

#ifndef CONFIG_SYS_ICACHE_OFF

void icache_enable(void)
{
	invalidate_icache_all();
	set_sctlr(get_sctlr() | CR_I);
}

void icache_disable(void)
{
	set_sctlr(get_sctlr() & ~CR_I);
}

int icache_status(void)
{
	return (get_sctlr() & CR_I) != 0;
}

void invalidate_icache_all(void)
{
	__asm_invalidate_icache_all();
	__asm_invalidate_l3_icache();
}

#else	/* CONFIG_SYS_ICACHE_OFF */

void icache_enable(void)
{
}

void icache_disable(void)
{
}

int icache_status(void)
{
	return 0;
}

void invalidate_icache_all(void)
{
}

#endif	/* CONFIG_SYS_ICACHE_OFF */

/*
 * Enable dCache & iCache, whether cache is actually enabled
 * depend on CONFIG_SYS_DCACHE_OFF and CONFIG_SYS_ICACHE_OFF
 */
void __weak enable_caches(void)
{
	icache_enable();
	dcache_enable();
}
Commit	Line	Data
0ae76531 DF	1	/*
	2	* (C) Copyright 2013
	3	* David Feng <fenghua@phytium.com.cn>
	4	*
5e2ec773 AG	5	* (C) Copyright 2016
	6	* Alexander Graf <agraf@suse.de>
	7	*
0ae76531 DF	8	* SPDX-License-Identifier: GPL-2.0+
	9	*/
	10
	11	#include <common.h>
	12	#include <asm/system.h>
	13	#include <asm/armv8/mmu.h>
	14
	15	DECLARE_GLOBAL_DATA_PTR;
	16
	17	#ifndef CONFIG_SYS_DCACHE_OFF
94f7ff36	18
5e2ec773 AG	19	/*
	20	* With 4k page granule, a virtual address is split into 4 lookup parts
	21	* spanning 9 bits each:
	22	*
	23	* _______________________________________________
	24	* \| \| \| \| \| \| \|
	25	* \| 0 \| Lv0 \| Lv1 \| Lv2 \| Lv3 \| off \|
	26	* \|_______\|_______\|_______\|_______\|_______\|_______\|
	27	* 63-48 47-39 38-30 29-21 20-12 11-00
	28	*
	29	* mask page size
	30	*
	31	* Lv0: FF8000000000 --
	32	* Lv1: 7FC0000000 1G
	33	* Lv2: 3FE00000 2M
	34	* Lv3: 1FF000 4K
	35	* off: FFF
	36	*/
94f7ff36	37
252cdb46	38	u64 get_tcr(int el, u64 pips, u64 pva_bits)
0691484a AG	39	{
	40	u64 max_addr = 0;
	41	u64 ips, va_bits;
	42	u64 tcr;
	43	int i;
	44
	45	/* Find the largest address we need to support */
d473f0c6	46	for (i = 0; mem_map[i].size \|\| mem_map[i].attrs; i++)
cd4b0c5f	47	max_addr = max(max_addr, mem_map[i].virt + mem_map[i].size);
0691484a AG	48
	49	/* Calculate the maximum physical (and thus virtual) address */
	50	if (max_addr > (1ULL << 44)) {
	51	ips = 5;
	52	va_bits = 48;
	53	} else if (max_addr > (1ULL << 42)) {
	54	ips = 4;
	55	va_bits = 44;
	56	} else if (max_addr > (1ULL << 40)) {
	57	ips = 3;
	58	va_bits = 42;
	59	} else if (max_addr > (1ULL << 36)) {
	60	ips = 2;
	61	va_bits = 40;
	62	} else if (max_addr > (1ULL << 32)) {
	63	ips = 1;
	64	va_bits = 36;
	65	} else {
	66	ips = 0;
	67	va_bits = 32;
	68	}
	69
	70	if (el == 1) {
9bb367a5	71	tcr = TCR_EL1_RSVD \| (ips << 32) \| TCR_EPD1_DISABLE;
0691484a AG	72	} else if (el == 2) {
	73	tcr = TCR_EL2_RSVD \| (ips << 16);
	74	} else {
	75	tcr = TCR_EL3_RSVD \| (ips << 16);
	76	}
	77
	78	/* PTWs cacheable, inner/outer WBWA and inner shareable */
5e2ec773 AG	79	tcr \|= TCR_TG0_4K \| TCR_SHARED_INNER \| TCR_ORGN_WBWA \| TCR_IRGN_WBWA;
5e2ec773 AG	80	tcr \|= TCR_T0SZ(va_bits);
0691484a AG	81
	82	if (pips)
	83	*pips = ips;
	84	if (pva_bits)
	85	*pva_bits = va_bits;
	86
	87	return tcr;
	88	}
	89
5e2ec773 AG	90	#define MAX_PTE_ENTRIES 512
	91
	92	static int pte_type(u64 *pte)
	93	{
	94	return *pte & PTE_TYPE_MASK;
	95	}
	96
	97	/* Returns the LSB number for a PTE on level <level> */
	98	static int level2shift(int level)
	99	{
	100	/* Page is 12 bits wide, every level translates 9 bits */
	101	return (12 + 9 * (3 - level));
	102	}
	103
	104	static u64 *find_pte(u64 addr, int level)
94f7ff36	105	{
5e2ec773 AG	106	int start_level = 0;
	107	u64 *pte;
	108	u64 idx;
	109	u64 va_bits;
	110	int i;
	111
	112	debug("addr=%llx level=%d\n", addr, level);
	113
	114	get_tcr(0, NULL, &va_bits);
	115	if (va_bits < 39)
	116	start_level = 1;
	117
	118	if (level < start_level)
	119	return NULL;
	120
	121	/* Walk through all page table levels to find our PTE */
	122	pte = (u64*)gd->arch.tlb_addr;
	123	for (i = start_level; i < 4; i++) {
	124	idx = (addr >> level2shift(i)) & 0x1FF;
	125	pte += idx;
	126	debug("idx=%llx PTE %p at level %d: %llx\n", idx, pte, i, *pte);
	127
	128	/* Found it */
	129	if (i == level)
	130	return pte;
	131	/* PTE is no table (either invalid or block), can't traverse */
	132	if (pte_type(pte) != PTE_TYPE_TABLE)
	133	return NULL;
	134	/* Off to the next level */
	135	pte = (u64)(pte & 0x0000fffffffff000ULL);
	136	}
	137
	138	/* Should never reach here */
	139	return NULL;
	140	}
	141
	142	/* Returns and creates a new full table (512 entries) */
	143	static u64 *create_table(void)
	144	{
	145	u64 new_table = (u64)gd->arch.tlb_fillptr;
	146	u64 pt_len = MAX_PTE_ENTRIES * sizeof(u64);
	147
	148	/* Allocate MAX_PTE_ENTRIES pte entries */
	149	gd->arch.tlb_fillptr += pt_len;
	150
	151	if (gd->arch.tlb_fillptr - gd->arch.tlb_addr > gd->arch.tlb_size)
	152	panic("Insufficient RAM for page table: 0x%lx > 0x%lx. "
	153	"Please increase the size in get_page_table_size()",
	154	gd->arch.tlb_fillptr - gd->arch.tlb_addr,
	155	gd->arch.tlb_size);
	156
	157	/* Mark all entries as invalid */
	158	memset(new_table, 0, pt_len);
	159
	160	return new_table;
	161	}
	162
	163	static void set_pte_table(u64 pte, u64 table)
	164	{
	165	/* Point pte to the new table /
	166	debug("Setting %p to addr=%p\n", pte, table);
	167	*pte = PTE_TYPE_TABLE \| (ulong)table;
	168	}
	169
f733d466 YS	170	/* Splits a block PTE into table with subpages spanning the old block */
	171	static void split_block(u64 *pte, int level)
	172	{
	173	u64 old_pte = *pte;
	174	u64 *new_table;
	175	u64 i = 0;
	176	/* level describes the parent level, we need the child ones */
	177	int levelshift = level2shift(level + 1);
	178
	179	if (pte_type(pte) != PTE_TYPE_BLOCK)
	180	panic("PTE %p (%llx) is not a block. Some driver code wants to "
	181	"modify dcache settings for an range not covered in "
	182	"mem_map.", pte, old_pte);
	183
	184	new_table = create_table();
	185	debug("Splitting pte %p (%llx) into %p\n", pte, old_pte, new_table);
	186
	187	for (i = 0; i < MAX_PTE_ENTRIES; i++) {
	188	new_table[i] = old_pte \| (i << levelshift);
	189
	190	/* Level 3 block PTEs have the table type */
	191	if ((level + 1) == 3)
	192	new_table[i] \|= PTE_TYPE_TABLE;
	193
	194	debug("Setting new_table[%lld] = %llx\n", i, new_table[i]);
	195	}
	196
	197	/* Set the new table into effect */
	198	set_pte_table(pte, new_table);
	199	}
	200
5e2ec773 AG	201	/* Add one mm_region map entry to the page tables */
	202	static void add_map(struct mm_region *map)
	203	{
	204	u64 *pte;
cd4b0c5f YS	205	u64 virt = map->virt;
cd4b0c5f YS	206	u64 phys = map->phys;
5e2ec773 AG	207	u64 size = map->size;
	208	u64 attrs = map->attrs \| PTE_TYPE_BLOCK \| PTE_BLOCK_AF;
	209	u64 blocksize;
	210	int level;
	211	u64 *new_table;
	212
	213	while (size) {
cd4b0c5f	214	pte = find_pte(virt, 0);
5e2ec773	215	if (pte && (pte_type(pte) == PTE_TYPE_FAULT)) {
cd4b0c5f	216	debug("Creating table for virt 0x%llx\n", virt);
5e2ec773 AG	217	new_table = create_table();
	218	set_pte_table(pte, new_table);
	219	}
	220
	221	for (level = 1; level < 4; level++) {
cd4b0c5f	222	pte = find_pte(virt, level);
f733d466 YS	223	if (!pte)
f733d466 YS	224	panic("pte not found\n");
cd4b0c5f	225
5e2ec773	226	blocksize = 1ULL << level2shift(level);
cd4b0c5f YS	227	debug("Checking if pte fits for virt=%llx size=%llx blocksize=%llx\n",
	228	virt, size, blocksize);
	229	if (size >= blocksize && !(virt & (blocksize - 1))) {
5e2ec773	230	/* Page fits, create block PTE */
cd4b0c5f YS	231	debug("Setting PTE %p to block virt=%llx\n",
	232	pte, virt);
	233	*pte = phys \| attrs;
	234	virt += blocksize;
	235	phys += blocksize;
5e2ec773 AG	236	size -= blocksize;
5e2ec773 AG	237	break;
f733d466	238	} else if (pte_type(pte) == PTE_TYPE_FAULT) {
5e2ec773	239	/* Page doesn't fit, create subpages */
cd4b0c5f YS	240	debug("Creating subtable for virt 0x%llx blksize=%llx\n",
cd4b0c5f YS	241	virt, blocksize);
5e2ec773 AG	242	new_table = create_table();
5e2ec773 AG	243	set_pte_table(pte, new_table);
f733d466	244	} else if (pte_type(pte) == PTE_TYPE_BLOCK) {
cd4b0c5f YS	245	debug("Split block into subtable for virt 0x%llx blksize=0x%llx\n",
cd4b0c5f YS	246	virt, blocksize);
f733d466	247	split_block(pte, level);
5e2ec773 AG	248	}
	249	}
	250	}
	251	}
	252
5e2ec773 AG	253	enum pte_type {
	254	PTE_INVAL,
	255	PTE_BLOCK,
	256	PTE_LEVEL,
	257	};
	258
	259	/*
	260	* This is a recursively called function to count the number of
	261	* page tables we need to cover a particular PTE range. If you
	262	* call this with level = -1 you basically get the full 48 bit
	263	* coverage.
	264	*/
	265	static int count_required_pts(u64 addr, int level, u64 maxaddr)
	266	{
	267	int levelshift = level2shift(level);
	268	u64 levelsize = 1ULL << levelshift;
	269	u64 levelmask = levelsize - 1;
	270	u64 levelend = addr + levelsize;
	271	int r = 0;
	272	int i;
	273	enum pte_type pte_type = PTE_INVAL;
	274
d473f0c6	275	for (i = 0; mem_map[i].size \|\| mem_map[i].attrs; i++) {
5e2ec773	276	struct mm_region *map = &mem_map[i];
cd4b0c5f	277	u64 start = map->virt;
5e2ec773 AG	278	u64 end = start + map->size;
	279
	280	/* Check if the PTE would overlap with the map */
	281	if (max(addr, start) <= min(levelend, end)) {
	282	start = max(addr, start);
	283	end = min(levelend, end);
	284
	285	/* We need a sub-pt for this level */
	286	if ((start & levelmask) \|\| (end & levelmask)) {
	287	pte_type = PTE_LEVEL;
	288	break;
94f7ff36 ST	289	}
94f7ff36 ST	290
5e2ec773 AG	291	/* Lv0 can not do block PTEs, so do levels here too */
	292	if (level <= 0) {
	293	pte_type = PTE_LEVEL;
	294	break;
	295	}
	296
	297	/* PTE is active, but fits into a block */
	298	pte_type = PTE_BLOCK;
94f7ff36 ST	299	}
94f7ff36 ST	300	}
5e2ec773 AG	301
	302	/*
	303	* Block PTEs at this level are already covered by the parent page
	304	* table, so we only need to count sub page tables.
	305	*/
	306	if (pte_type == PTE_LEVEL) {
	307	int sublevel = level + 1;
	308	u64 sublevelsize = 1ULL << level2shift(sublevel);
	309
	310	/* Account for the new sub page table ... */
	311	r = 1;
	312
	313	/* ... and for all child page tables that one might have */
	314	for (i = 0; i < MAX_PTE_ENTRIES; i++) {
	315	r += count_required_pts(addr, sublevel, maxaddr);
	316	addr += sublevelsize;
	317
	318	if (addr >= maxaddr) {
	319	/*
	320	* We reached the end of address space, no need
	321	* to look any further.
	322	*/
	323	break;
	324	}
	325	}
	326	}
	327
	328	return r;
	329	}
	330
	331	/* Returns the estimated required size of all page tables */
c05016ab	332	__weak u64 get_page_table_size(void)
5e2ec773 AG	333	{
	334	u64 one_pt = MAX_PTE_ENTRIES * sizeof(u64);
	335	u64 size = 0;
	336	u64 va_bits;
	337	int start_level = 0;
	338
	339	get_tcr(0, NULL, &va_bits);
	340	if (va_bits < 39)
	341	start_level = 1;
	342
	343	/* Account for all page tables we would need to cover our memory map */
	344	size = one_pt * count_required_pts(0, start_level - 1, 1ULL << va_bits);
	345
	346	/*
	347	* We need to duplicate our page table once to have an emergency pt to
	348	* resort to when splitting page tables later on
	349	*/
	350	size *= 2;
	351
	352	/*
	353	* We may need to split page tables later on if dcache settings change,
	354	* so reserve up to 4 (random pick) page tables for that.
	355	*/
	356	size += one_pt * 4;
	357
	358	return size;
	359	}
	360
252cdb46	361	void setup_pgtables(void)
5e2ec773 AG	362	{
	363	int i;
	364
252cdb46 YS	365	if (!gd->arch.tlb_fillptr \|\| !gd->arch.tlb_addr)
	366	panic("Page table pointer not setup.");
	367
5e2ec773 AG	368	/*
	369	* Allocate the first level we're on with invalidate entries.
	370	* If the starting level is 0 (va_bits >= 39), then this is our
	371	* Lv0 page table, otherwise it's the entry Lv1 page table.
	372	*/
	373	create_table();
	374
	375	/* Now add all MMU table entries one after another to the table */
d473f0c6	376	for (i = 0; mem_map[i].size \|\| mem_map[i].attrs; i++)
5e2ec773	377	add_map(&mem_map[i]);
5e2ec773 AG	378	}
	379
	380	static void setup_all_pgtables(void)
	381	{
	382	u64 tlb_addr = gd->arch.tlb_addr;
0e170947	383	u64 tlb_size = gd->arch.tlb_size;
5e2ec773 AG	384
	385	/* Reset the fill ptr */
	386	gd->arch.tlb_fillptr = tlb_addr;
	387
	388	/* Create normal system page tables */
	389	setup_pgtables();
	390
	391	/* Create emergency page tables */
0e170947 AG	392	gd->arch.tlb_size -= (uintptr_t)gd->arch.tlb_fillptr -
0e170947 AG	393	(uintptr_t)gd->arch.tlb_addr;
5e2ec773 AG	394	gd->arch.tlb_addr = gd->arch.tlb_fillptr;
	395	setup_pgtables();
	396	gd->arch.tlb_emerg = gd->arch.tlb_addr;
	397	gd->arch.tlb_addr = tlb_addr;
0e170947	398	gd->arch.tlb_size = tlb_size;
94f7ff36 ST	399	}
94f7ff36 ST	400
0ae76531	401	/* to activate the MMU we need to set up virtual memory */
3c6af3ba	402	__weak void mmu_setup(void)
0ae76531	403	{
8b19dff5	404	int el;
0ae76531	405
5e2ec773 AG	406	/* Set up page tables only once */
	407	if (!gd->arch.tlb_fillptr)
	408	setup_all_pgtables();
0691484a AG	409
	410	el = current_el();
	411	set_ttbr_tcr_mair(el, gd->arch.tlb_addr, get_tcr(el, NULL, NULL),
	412	MEMORY_ATTRIBUTES);
0691484a	413
0ae76531 DF	414	/* enable the mmu */
	415	set_sctlr(get_sctlr() \| CR_M);
	416	}
	417
	418	/*
	419	* Performs a invalidation of the entire data cache at all levels
	420	*/
	421	void invalidate_dcache_all(void)
	422	{
1e6ad55c	423	__asm_invalidate_dcache_all();
1ab557a0	424	__asm_invalidate_l3_dcache();
0ae76531 DF	425	}
	426
	427	/*
dcd468b8 YS	428	* Performs a clean & invalidation of the entire data cache at all levels.
dcd468b8 YS	429	* This function needs to be inline to avoid using stack.
1ab557a0	430	* __asm_flush_l3_dcache return status of timeout
0ae76531	431	*/
dcd468b8	432	inline void flush_dcache_all(void)
0ae76531	433	{
dcd468b8 YS	434	int ret;
dcd468b8 YS	435
0ae76531	436	__asm_flush_dcache_all();
1ab557a0	437	ret = __asm_flush_l3_dcache();
dcd468b8 YS	438	if (ret)
	439	debug("flushing dcache returns 0x%x\n", ret);
	440	else
	441	debug("flushing dcache successfully.\n");
0ae76531 DF	442	}
	443
	444	/*
	445	* Invalidates range in all levels of D-cache/unified cache
	446	*/
	447	void invalidate_dcache_range(unsigned long start, unsigned long stop)
	448	{
	449	__asm_flush_dcache_range(start, stop);
	450	}
	451
	452	/*
	453	* Flush range(clean & invalidate) from all levels of D-cache/unified cache
	454	*/
	455	void flush_dcache_range(unsigned long start, unsigned long stop)
	456	{
	457	__asm_flush_dcache_range(start, stop);
	458	}
	459
	460	void dcache_enable(void)
	461	{
	462	/* The data cache is not active unless the mmu is enabled */
	463	if (!(get_sctlr() & CR_M)) {
	464	invalidate_dcache_all();
	465	__asm_invalidate_tlb_all();
	466	mmu_setup();
	467	}
	468
	469	set_sctlr(get_sctlr() \| CR_C);
	470	}
	471
	472	void dcache_disable(void)
	473	{
	474	uint32_t sctlr;
	475
	476	sctlr = get_sctlr();
	477
	478	/* if cache isn't enabled no need to disable */
	479	if (!(sctlr & CR_C))
	480	return;
	481
	482	set_sctlr(sctlr & ~(CR_C\|CR_M));
	483
	484	flush_dcache_all();
	485	__asm_invalidate_tlb_all();
	486	}
	487
	488	int dcache_status(void)
	489	{
	490	return (get_sctlr() & CR_C) != 0;
	491	}
	492
dad17fd5 SDPP	493	u64 *__weak arch_get_page_table(void) {
	494	puts("No page table offset defined\n");
	495
	496	return NULL;
	497	}
	498
5e2ec773 AG	499	static bool is_aligned(u64 addr, u64 size, u64 align)
	500	{
	501	return !(addr & (align - 1)) && !(size & (align - 1));
	502	}
	503
	504	static u64 set_one_region(u64 start, u64 size, u64 attrs, int level)
	505	{
	506	int levelshift = level2shift(level);
	507	u64 levelsize = 1ULL << levelshift;
	508	u64 *pte = find_pte(start, level);
	509
	510	/* Can we can just modify the current level block PTE? */
	511	if (is_aligned(start, size, levelsize)) {
	512	*pte &= ~PMD_ATTRINDX_MASK;
	513	*pte \|= attrs;
	514	debug("Set attrs=%llx pte=%p level=%d\n", attrs, pte, level);
	515
	516	return levelsize;
	517	}
	518
	519	/* Unaligned or doesn't fit, maybe split block into table */
	520	debug("addr=%llx level=%d pte=%p (%llx)\n", start, level, pte, *pte);
	521
	522	/* Maybe we need to split the block into a table */
	523	if (pte_type(pte) == PTE_TYPE_BLOCK)
	524	split_block(pte, level);
	525
	526	/* And then double-check it became a table or already is one */
	527	if (pte_type(pte) != PTE_TYPE_TABLE)
	528	panic("PTE %p (%llx) for addr=%llx should be a table",
	529	pte, *pte, start);
	530
	531	/* Roll on to the next page table level */
	532	return 0;
	533	}
	534
	535	void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size,
	536	enum dcache_option option)
	537	{
	538	u64 attrs = PMD_ATTRINDX(option);
	539	u64 real_start = start;
	540	u64 real_size = size;
	541
	542	debug("start=%lx size=%lx\n", (ulong)start, (ulong)size);
	543
252cdb46 YS	544	if (!gd->arch.tlb_emerg)
	545	panic("Emergency page table not setup.");
	546
5e2ec773 AG	547	/*
	548	* We can not modify page tables that we're currently running on,
	549	* so we first need to switch to the "emergency" page tables where
	550	* we can safely modify our primary page tables and then switch back
	551	*/
	552	__asm_switch_ttbr(gd->arch.tlb_emerg);
	553
	554	/*
	555	* Loop through the address range until we find a page granule that fits
	556	* our alignment constraints, then set it to the new cache attributes
	557	*/
	558	while (size > 0) {
	559	int level;
	560	u64 r;
	561
	562	for (level = 1; level < 4; level++) {
	563	r = set_one_region(start, size, attrs, level);
	564	if (r) {
	565	/* PTE successfully replaced */
	566	size -= r;
	567	start += r;
	568	break;
	569	}
	570	}
	571
	572	}
	573
	574	/* We're done modifying page tables, switch back to our primary ones */
	575	__asm_switch_ttbr(gd->arch.tlb_addr);
	576
	577	/*
	578	* Make sure there's nothing stale in dcache for a region that might
	579	* have caches off now
	580	*/
	581	flush_dcache_range(real_start, real_start + real_size);
	582	}
94f7ff36	583
0ae76531 DF	584	#else /* CONFIG_SYS_DCACHE_OFF */
0ae76531 DF	585
19503c31 AG	586	/*
	587	* For SPL builds, we may want to not have dcache enabled. Any real U-Boot
	588	* running however really wants to have dcache and the MMU active. Check that
	589	* everything is sane and give the developer a hint if it isn't.
	590	*/
	591	#ifndef CONFIG_SPL_BUILD
	592	#error Please describe your MMU layout in CONFIG_SYS_MEM_MAP and enable dcache.
	593	#endif
	594
0ae76531 DF	595	void invalidate_dcache_all(void)
	596	{
	597	}
	598
	599	void flush_dcache_all(void)
	600	{
	601	}
	602
0ae76531 DF	603	void dcache_enable(void)
	604	{
	605	}
	606
	607	void dcache_disable(void)
	608	{
	609	}
	610
	611	int dcache_status(void)
	612	{
	613	return 0;
	614	}
	615
dad17fd5 SDPP	616	void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size,
	617	enum dcache_option option)
	618	{
	619	}
	620
0ae76531 DF	621	#endif /* CONFIG_SYS_DCACHE_OFF */
	622
	623	#ifndef CONFIG_SYS_ICACHE_OFF
	624
	625	void icache_enable(void)
	626	{
1ab557a0	627	invalidate_icache_all();
0ae76531 DF	628	set_sctlr(get_sctlr() \| CR_I);
	629	}
	630
	631	void icache_disable(void)
	632	{
	633	set_sctlr(get_sctlr() & ~CR_I);
	634	}
	635
	636	int icache_status(void)
	637	{
	638	return (get_sctlr() & CR_I) != 0;
	639	}
	640
	641	void invalidate_icache_all(void)
	642	{
	643	__asm_invalidate_icache_all();
1ab557a0	644	__asm_invalidate_l3_icache();
0ae76531 DF	645	}
	646
	647	#else /* CONFIG_SYS_ICACHE_OFF */
	648
	649	void icache_enable(void)
	650	{
	651	}
	652
	653	void icache_disable(void)
	654	{
	655	}
	656
	657	int icache_status(void)
	658	{
	659	return 0;
	660	}
	661
	662	void invalidate_icache_all(void)
	663	{
	664	}
	665
	666	#endif /* CONFIG_SYS_ICACHE_OFF */
	667
	668	/*
	669	* Enable dCache & iCache, whether cache is actually enabled
	670	* depend on CONFIG_SYS_DCACHE_OFF and CONFIG_SYS_ICACHE_OFF
	671	*/
2f78eae5	672	void __weak enable_caches(void)
0ae76531 DF	673	{
	674	icache_enable();
	675	dcache_enable();
	676	}