[people/ms/u-boot.git] / arch / arc / lib / cache.c

/*
 * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved.
 *
 * SPDX-License-Identifier:	GPL-2.0+
 */

#include <config.h>
#include <common.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/log2.h>
#include <asm/arcregs.h>
#include <asm/cache.h>

/* Bit values in IC_CTRL */
#define IC_CTRL_CACHE_DISABLE	(1 << 0)

/* Bit values in DC_CTRL */
#define DC_CTRL_CACHE_DISABLE	(1 << 0)
#define DC_CTRL_INV_MODE_FLUSH	(1 << 6)
#define DC_CTRL_FLUSH_STATUS	(1 << 8)
#define CACHE_VER_NUM_MASK	0xF

#define OP_INV		0x1
#define OP_FLUSH	0x2
#define OP_INV_IC	0x3

/* Bit val in SLC_CONTROL */
#define SLC_CTRL_DIS		0x001
#define SLC_CTRL_IM		0x040
#define SLC_CTRL_BUSY		0x100
#define SLC_CTRL_RGN_OP_INV	0x200

/*
 * By default that variable will fall into .bss section.
 * But .bss section is not relocated and so it will be initilized before
 * relocation but will be used after being zeroed.
 */
int l1_line_sz __section(".data");
bool dcache_exists __section(".data") = false;
bool icache_exists __section(".data") = false;

#define CACHE_LINE_MASK		(~(l1_line_sz - 1))

#ifdef CONFIG_ISA_ARCV2
int slc_line_sz __section(".data");
bool slc_exists __section(".data") = false;
bool ioc_exists __section(".data") = false;
bool pae_exists __section(".data") = false;

void read_decode_mmu_bcr(void)
{
	/* TODO: should we compare mmu version from BCR and from CONFIG? */
#if (CONFIG_ARC_MMU_VER >= 4)
	u32 tmp;

	tmp = read_aux_reg(ARC_AUX_MMU_BCR);

	struct bcr_mmu_4 {
#ifdef CONFIG_CPU_BIG_ENDIAN
	unsigned int ver:8, sasid:1, sz1:4, sz0:4, res:2, pae:1,
		     n_ways:2, n_entry:2, n_super:2, u_itlb:3, u_dtlb:3;
#else
	/*           DTLB      ITLB      JES        JE         JA      */
	unsigned int u_dtlb:3, u_itlb:3, n_super:2, n_entry:2, n_ways:2,
		     pae:1, res:2, sz0:4, sz1:4, sasid:1, ver:8;
#endif /* CONFIG_CPU_BIG_ENDIAN */
	} *mmu4;

	mmu4 = (struct bcr_mmu_4 *)&tmp;

	pae_exists = !!mmu4->pae;
#endif /* (CONFIG_ARC_MMU_VER >= 4) */
}

static void __slc_entire_op(const int op)
{
	unsigned int ctrl;

	ctrl = read_aux_reg(ARC_AUX_SLC_CTRL);

	if (!(op & OP_FLUSH))		/* i.e. OP_INV */
		ctrl &= ~SLC_CTRL_IM;	/* clear IM: Disable flush before Inv */
	else
		ctrl |= SLC_CTRL_IM;

	write_aux_reg(ARC_AUX_SLC_CTRL, ctrl);

	if (op & OP_INV)	/* Inv or flush-n-inv use same cmd reg */
		write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1);
	else
		write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1);

	/* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
	read_aux_reg(ARC_AUX_SLC_CTRL);

	/* Important to wait for flush to complete */
	while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY);
}

static void slc_upper_region_init(void)
{
	/*
	 * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0
	 * as we don't use PAE40.
	 */
	write_aux_reg(ARC_AUX_SLC_RGN_END1, 0);
	write_aux_reg(ARC_AUX_SLC_RGN_START1, 0);
}

static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op)
{
	unsigned int ctrl;
	unsigned long end;

	/*
	 * The Region Flush operation is specified by CTRL.RGN_OP[11..9]
	 *  - b'000 (default) is Flush,
	 *  - b'001 is Invalidate if CTRL.IM == 0
	 *  - b'001 is Flush-n-Invalidate if CTRL.IM == 1
	 */
	ctrl = read_aux_reg(ARC_AUX_SLC_CTRL);

	/* Don't rely on default value of IM bit */
	if (!(op & OP_FLUSH))		/* i.e. OP_INV */
		ctrl &= ~SLC_CTRL_IM;	/* clear IM: Disable flush before Inv */
	else
		ctrl |= SLC_CTRL_IM;

	if (op & OP_INV)
		ctrl |= SLC_CTRL_RGN_OP_INV;	/* Inv or flush-n-inv */
	else
		ctrl &= ~SLC_CTRL_RGN_OP_INV;

	write_aux_reg(ARC_AUX_SLC_CTRL, ctrl);

	/*
	 * Lower bits are ignored, no need to clip
	 * END needs to be setup before START (latter triggers the operation)
	 * END can't be same as START, so add (l2_line_sz - 1) to sz
	 */
	end = paddr + sz + slc_line_sz - 1;

	/*
	 * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1)
	 * are always == 0 as we don't use PAE40, so we only setup lower ones
	 * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START)
	 */
	write_aux_reg(ARC_AUX_SLC_RGN_END, end);
	write_aux_reg(ARC_AUX_SLC_RGN_START, paddr);

	/* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
	read_aux_reg(ARC_AUX_SLC_CTRL);

	while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY);
}
#endif /* CONFIG_ISA_ARCV2 */

#ifdef CONFIG_ISA_ARCV2
static void read_decode_cache_bcr_arcv2(void)
{
	union {
		struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:24, way:2, lsz:2, sz:4;
#else
			unsigned int sz:4, lsz:2, way:2, pad:24;
#endif
		} fields;
		unsigned int word;
	} slc_cfg;

	union {
		struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:24, ver:8;
#else
			unsigned int ver:8, pad:24;
#endif
		} fields;
		unsigned int word;
	} sbcr;

	sbcr.word = read_aux_reg(ARC_BCR_SLC);
	if (sbcr.fields.ver) {
		slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG);
		slc_exists = true;
		slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64;
	}

	union {
		struct bcr_clust_cfg {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8;
#else
			unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7;
#endif
		} fields;
		unsigned int word;
	} cbcr;

	cbcr.word = read_aux_reg(ARC_BCR_CLUSTER);
	if (cbcr.fields.c)
		ioc_exists = true;
}
#endif

void read_decode_cache_bcr(void)
{
	int dc_line_sz = 0, ic_line_sz = 0;

	union {
		struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
#else
			unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
#endif
		} fields;
		unsigned int word;
	} ibcr, dbcr;

	ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
	if (ibcr.fields.ver) {
		icache_exists = true;
		l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len;
		if (!ic_line_sz)
			panic("Instruction exists but line length is 0\n");
	}

	dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
	if (dbcr.fields.ver){
		dcache_exists = true;
		l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len;
		if (!dc_line_sz)
			panic("Data cache exists but line length is 0\n");
	}

	if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz))
		panic("Instruction and data cache line lengths differ\n");
}

void cache_init(void)
{
	read_decode_cache_bcr();

#ifdef CONFIG_ISA_ARCV2
	read_decode_cache_bcr_arcv2();

	if (ioc_exists) {
		/* IOC Aperture start is equal to DDR start */
		unsigned int ap_base = CONFIG_SYS_SDRAM_BASE;
		/* IOC Aperture size is equal to DDR size */
		long ap_size = CONFIG_SYS_SDRAM_SIZE;

		flush_dcache_all();
		invalidate_dcache_all();

		if (!is_power_of_2(ap_size) || ap_size < 4096)
			panic("IOC Aperture size must be power of 2 and bigger 4Kib");

		/*
		 * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB,
		 * so setting 0x11 implies 512M, 0x12 implies 1G...
		 */
		write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE,
			      order_base_2(ap_size/1024) - 2);


		/* IOC Aperture start must be aligned to the size of the aperture */
		if (ap_base % ap_size != 0)
			panic("IOC Aperture start must be aligned to the size of the aperture");

		write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12);
		write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1);
		write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1);

	}

	read_decode_mmu_bcr();

	/*
	 * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist
	 * only if PAE exists in current HW. So we had to check pae_exist
	 * before using them.
	 */
	if (slc_exists && pae_exists)
		slc_upper_region_init();
#endif /* CONFIG_ISA_ARCV2 */
}

int icache_status(void)
{
	if (!icache_exists)
		return 0;

	if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE)
		return 0;
	else
		return 1;
}

void icache_enable(void)
{
	if (icache_exists)
		write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) &
			      ~IC_CTRL_CACHE_DISABLE);
}

void icache_disable(void)
{
	if (icache_exists)
		write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) |
			      IC_CTRL_CACHE_DISABLE);
}

void invalidate_icache_all(void)
{
	/* Any write to IC_IVIC register triggers invalidation of entire I$ */
	if (icache_status()) {
		write_aux_reg(ARC_AUX_IC_IVIC, 1);
		/*
		 * As per ARC HS databook (see chapter 5.3.3.2)
		 * it is required to add 3 NOPs after each write to IC_IVIC.
		 */
		__builtin_arc_nop();
		__builtin_arc_nop();
		__builtin_arc_nop();
		read_aux_reg(ARC_AUX_IC_CTRL);	/* blocks */
	}

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists)
		__slc_entire_op(OP_INV);
#endif
}

int dcache_status(void)
{
	if (!dcache_exists)
		return 0;

	if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE)
		return 0;
	else
		return 1;
}

void dcache_enable(void)
{
	if (!dcache_exists)
		return;

	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) &
		      ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE));
}

void dcache_disable(void)
{
	if (!dcache_exists)
		return;

	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) |
		      DC_CTRL_CACHE_DISABLE);
}

#ifndef CONFIG_SYS_DCACHE_OFF
/*
 * Common Helper for Line Operations on {I,D}-Cache
 */
static inline void __cache_line_loop(unsigned long paddr, unsigned long sz,
				     const int cacheop)
{
	unsigned int aux_cmd;
#if (CONFIG_ARC_MMU_VER == 3)
	unsigned int aux_tag;
#endif
	int num_lines;

	if (cacheop == OP_INV_IC) {
		aux_cmd = ARC_AUX_IC_IVIL;
#if (CONFIG_ARC_MMU_VER == 3)
		aux_tag = ARC_AUX_IC_PTAG;
#endif
	} else {
		/* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
		aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL;
#if (CONFIG_ARC_MMU_VER == 3)
		aux_tag = ARC_AUX_DC_PTAG;
#endif
	}

	sz += paddr & ~CACHE_LINE_MASK;
	paddr &= CACHE_LINE_MASK;

	num_lines = DIV_ROUND_UP(sz, l1_line_sz);

	while (num_lines-- > 0) {
#if (CONFIG_ARC_MMU_VER == 3)
		write_aux_reg(aux_tag, paddr);
#endif
		write_aux_reg(aux_cmd, paddr);
		paddr += l1_line_sz;
	}
}

static unsigned int __before_dc_op(const int op)
{
	unsigned int reg;

	if (op == OP_INV) {
		/*
		 * IM is set by default and implies Flush-n-inv
		 * Clear it here for vanilla inv
		 */
		reg = read_aux_reg(ARC_AUX_DC_CTRL);
		write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
	}

	return reg;
}

static void __after_dc_op(const int op, unsigned int reg)
{
	if (op & OP_FLUSH)	/* flush / flush-n-inv both wait */
		while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
			;

	/* Switch back to default Invalidate mode */
	if (op == OP_INV)
		write_aux_reg(ARC_AUX_DC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH);
}

static inline void __dc_entire_op(const int cacheop)
{
	int aux;
	unsigned int ctrl_reg = __before_dc_op(cacheop);

	if (cacheop & OP_INV)	/* Inv or flush-n-inv use same cmd reg */
		aux = ARC_AUX_DC_IVDC;
	else
		aux = ARC_AUX_DC_FLSH;

	write_aux_reg(aux, 0x1);

	__after_dc_op(cacheop, ctrl_reg);
}

static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
				const int cacheop)
{
	unsigned int ctrl_reg = __before_dc_op(cacheop);
	__cache_line_loop(paddr, sz, cacheop);
	__after_dc_op(cacheop, ctrl_reg);
}
#else
#define __dc_entire_op(cacheop)
#define __dc_line_op(paddr, sz, cacheop)
#endif /* !CONFIG_SYS_DCACHE_OFF */

void invalidate_dcache_range(unsigned long start, unsigned long end)
{
	if (start >= end)
		return;

#ifdef CONFIG_ISA_ARCV2
	if (!ioc_exists)
#endif
		__dc_line_op(start, end - start, OP_INV);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists && !ioc_exists)
		__slc_rgn_op(start, end - start, OP_INV);
#endif
}

void flush_dcache_range(unsigned long start, unsigned long end)
{
	if (start >= end)
		return;

#ifdef CONFIG_ISA_ARCV2
	if (!ioc_exists)
#endif
		__dc_line_op(start, end - start, OP_FLUSH);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists && !ioc_exists)
		__slc_rgn_op(start, end - start, OP_FLUSH);
#endif
}

void flush_cache(unsigned long start, unsigned long size)
{
	flush_dcache_range(start, start + size);
}

void invalidate_dcache_all(void)
{
	__dc_entire_op(OP_INV);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists)
		__slc_entire_op(OP_INV);
#endif
}

void flush_dcache_all(void)
{
	__dc_entire_op(OP_FLUSH);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists)
		__slc_entire_op(OP_FLUSH);
#endif
}
Commit	Line	Data
2f16ac9d AB	1	/*
	2	* Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved.
	3	*
	4	* SPDX-License-Identifier: GPL-2.0+
	5	*/
	6
	7	#include <config.h>
379b3280	8	#include <common.h>
ef639e6f AB	9	#include <linux/compiler.h>
ef639e6f AB	10	#include <linux/kernel.h>
97a63144	11	#include <linux/log2.h>
2f16ac9d	12	#include <asm/arcregs.h>
205e7a7b	13	#include <asm/cache.h>
2f16ac9d AB	14
	15	/* Bit values in IC_CTRL */
	16	#define IC_CTRL_CACHE_DISABLE (1 << 0)
	17
	18	/* Bit values in DC_CTRL */
	19	#define DC_CTRL_CACHE_DISABLE (1 << 0)
	20	#define DC_CTRL_INV_MODE_FLUSH (1 << 6)
	21	#define DC_CTRL_FLUSH_STATUS (1 << 8)
f8cf3d1e	22	#define CACHE_VER_NUM_MASK 0xF
2f16ac9d	23
ef639e6f AB	24	#define OP_INV 0x1
	25	#define OP_FLUSH 0x2
	26	#define OP_INV_IC 0x3
	27
41cada4d EP	28	/* Bit val in SLC_CONTROL */
	29	#define SLC_CTRL_DIS 0x001
	30	#define SLC_CTRL_IM 0x040
	31	#define SLC_CTRL_BUSY 0x100
	32	#define SLC_CTRL_RGN_OP_INV 0x200
	33
ef639e6f AB	34	/*
	35	* By default that variable will fall into .bss section.
	36	* But .bss section is not relocated and so it will be initilized before
	37	* relocation but will be used after being zeroed.
	38	*/
379b3280	39	int l1_line_sz __section(".data");
3cf23939 EP	40	bool dcache_exists __section(".data") = false;
3cf23939 EP	41	bool icache_exists __section(".data") = false;
379b3280 AB	42
	43	#define CACHE_LINE_MASK (~(l1_line_sz - 1))
	44
	45	#ifdef CONFIG_ISA_ARCV2
ef639e6f	46	int slc_line_sz __section(".data");
3cf23939 EP	47	bool slc_exists __section(".data") = false;
3cf23939 EP	48	bool ioc_exists __section(".data") = false;
41cada4d	49	bool pae_exists __section(".data") = false;
ef639e6f	50
41cada4d	51	void read_decode_mmu_bcr(void)
ef639e6f	52	{
41cada4d EP	53	/* TODO: should we compare mmu version from BCR and from CONFIG? */
	54	#if (CONFIG_ARC_MMU_VER >= 4)
	55	u32 tmp;
ef639e6f	56
41cada4d	57	tmp = read_aux_reg(ARC_AUX_MMU_BCR);
ef639e6f	58
41cada4d EP	59	struct bcr_mmu_4 {
	60	#ifdef CONFIG_CPU_BIG_ENDIAN
	61	unsigned int ver:8, sasid:1, sz1:4, sz0:4, res:2, pae:1,
	62	n_ways:2, n_entry:2, n_super:2, u_itlb:3, u_dtlb:3;
	63	#else
	64	/* DTLB ITLB JES JE JA */
	65	unsigned int u_dtlb:3, u_itlb:3, n_super:2, n_entry:2, n_ways:2,
	66	pae:1, res:2, sz0:4, sz1:4, sasid:1, ver:8;
	67	#endif /* CONFIG_CPU_BIG_ENDIAN */
	68	} *mmu4;
ef639e6f	69
41cada4d	70	mmu4 = (struct bcr_mmu_4 *)&tmp;
ef639e6f	71
41cada4d EP	72	pae_exists = !!mmu4->pae;
41cada4d EP	73	#endif /* (CONFIG_ARC_MMU_VER >= 4) */
ef639e6f AB	74	}
ef639e6f AB	75
41cada4d	76	static void __slc_entire_op(const int op)
ef639e6f	77	{
41cada4d EP	78	unsigned int ctrl;
	79
	80	ctrl = read_aux_reg(ARC_AUX_SLC_CTRL);
ef639e6f	81
41cada4d EP	82	if (!(op & OP_FLUSH)) /* i.e. OP_INV */
	83	ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
	84	else
	85	ctrl \|= SLC_CTRL_IM;
ef639e6f	86
41cada4d	87	write_aux_reg(ARC_AUX_SLC_CTRL, ctrl);
ef639e6f	88
41cada4d EP	89	if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */
	90	write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1);
	91	else
	92	write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1);
ef639e6f	93
41cada4d EP	94	/* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
41cada4d EP	95	read_aux_reg(ARC_AUX_SLC_CTRL);
ef639e6f	96
41cada4d EP	97	/* Important to wait for flush to complete */
41cada4d EP	98	while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY);
ef639e6f AB	99	}
ef639e6f AB	100
41cada4d	101	static void slc_upper_region_init(void)
ef639e6f	102	{
41cada4d EP	103	/*
	104	* ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0
	105	* as we don't use PAE40.
	106	*/
	107	write_aux_reg(ARC_AUX_SLC_RGN_END1, 0);
	108	write_aux_reg(ARC_AUX_SLC_RGN_START1, 0);
	109	}
ef639e6f	110
41cada4d EP	111	static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op)
	112	{
	113	unsigned int ctrl;
	114	unsigned long end;
	115
	116	/*
	117	* The Region Flush operation is specified by CTRL.RGN_OP[11..9]
	118	* - b'000 (default) is Flush,
	119	* - b'001 is Invalidate if CTRL.IM == 0
	120	* - b'001 is Flush-n-Invalidate if CTRL.IM == 1
	121	*/
	122	ctrl = read_aux_reg(ARC_AUX_SLC_CTRL);
	123
	124	/* Don't rely on default value of IM bit */
	125	if (!(op & OP_FLUSH)) /* i.e. OP_INV */
	126	ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
ef639e6f	127	else
41cada4d	128	ctrl \|= SLC_CTRL_IM;
ef639e6f	129
41cada4d EP	130	if (op & OP_INV)
	131	ctrl \|= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */
	132	else
	133	ctrl &= ~SLC_CTRL_RGN_OP_INV;
ef639e6f	134
41cada4d	135	write_aux_reg(ARC_AUX_SLC_CTRL, ctrl);
ef639e6f	136
41cada4d EP	137	/*
	138	* Lower bits are ignored, no need to clip
	139	* END needs to be setup before START (latter triggers the operation)
	140	* END can't be same as START, so add (l2_line_sz - 1) to sz
	141	*/
	142	end = paddr + sz + slc_line_sz - 1;
	143
	144	/*
	145	* Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1)
	146	* are always == 0 as we don't use PAE40, so we only setup lower ones
	147	* (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START)
	148	*/
	149	write_aux_reg(ARC_AUX_SLC_RGN_END, end);
	150	write_aux_reg(ARC_AUX_SLC_RGN_START, paddr);
	151
	152	/* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
	153	read_aux_reg(ARC_AUX_SLC_CTRL);
	154
	155	while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY);
ef639e6f	156	}
41cada4d	157	#endif /* CONFIG_ISA_ARCV2 */
ef639e6f	158
379b3280 AB	159	#ifdef CONFIG_ISA_ARCV2
379b3280 AB	160	static void read_decode_cache_bcr_arcv2(void)
ef639e6f	161	{
379b3280 AB	162	union {
	163	struct {
	164	#ifdef CONFIG_CPU_BIG_ENDIAN
	165	unsigned int pad:24, way:2, lsz:2, sz:4;
	166	#else
	167	unsigned int sz:4, lsz:2, way:2, pad:24;
	168	#endif
	169	} fields;
	170	unsigned int word;
	171	} slc_cfg;
	172
	173	union {
	174	struct {
	175	#ifdef CONFIG_CPU_BIG_ENDIAN
	176	unsigned int pad:24, ver:8;
	177	#else
	178	unsigned int ver:8, pad:24;
	179	#endif
	180	} fields;
	181	unsigned int word;
	182	} sbcr;
	183
	184	sbcr.word = read_aux_reg(ARC_BCR_SLC);
	185	if (sbcr.fields.ver) {
	186	slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG);
3cf23939	187	slc_exists = true;
379b3280 AB	188	slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64;
379b3280 AB	189	}
db6ce231 AB	190
	191	union {
	192	struct bcr_clust_cfg {
	193	#ifdef CONFIG_CPU_BIG_ENDIAN
	194	unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8;
	195	#else
	196	unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7;
	197	#endif
	198	} fields;
	199	unsigned int word;
	200	} cbcr;
	201
	202	cbcr.word = read_aux_reg(ARC_BCR_CLUSTER);
	203	if (cbcr.fields.c)
3cf23939	204	ioc_exists = true;
ef639e6f	205	}
379b3280	206	#endif
ef639e6f	207
379b3280	208	void read_decode_cache_bcr(void)
ef639e6f	209	{
379b3280 AB	210	int dc_line_sz = 0, ic_line_sz = 0;
	211
	212	union {
	213	struct {
	214	#ifdef CONFIG_CPU_BIG_ENDIAN
	215	unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
	216	#else
	217	unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
	218	#endif
	219	} fields;
	220	unsigned int word;
	221	} ibcr, dbcr;
	222
	223	ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
	224	if (ibcr.fields.ver) {
3cf23939	225	icache_exists = true;
379b3280 AB	226	l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len;
	227	if (!ic_line_sz)
	228	panic("Instruction exists but line length is 0\n");
	229	}
	230
	231	dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
	232	if (dbcr.fields.ver){
3cf23939	233	dcache_exists = true;
379b3280 AB	234	l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len;
	235	if (!dc_line_sz)
	236	panic("Data cache exists but line length is 0\n");
	237	}
	238
	239	if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz))
	240	panic("Instruction and data cache line lengths differ\n");
ef639e6f AB	241	}
	242
	243	void cache_init(void)
	244	{
379b3280 AB	245	read_decode_cache_bcr();
379b3280 AB	246
ef639e6f	247	#ifdef CONFIG_ISA_ARCV2
379b3280	248	read_decode_cache_bcr_arcv2();
db6ce231 AB	249
db6ce231 AB	250	if (ioc_exists) {
97a63144 AB	251	/* IOC Aperture start is equal to DDR start */
	252	unsigned int ap_base = CONFIG_SYS_SDRAM_BASE;
	253	/* IOC Aperture size is equal to DDR size */
	254	long ap_size = CONFIG_SYS_SDRAM_SIZE;
	255
a4a43fcf AB	256	flush_dcache_all();
	257	invalidate_dcache_all();
	258
97a63144 AB	259	if (!is_power_of_2(ap_size) \|\| ap_size < 4096)
	260	panic("IOC Aperture size must be power of 2 and bigger 4Kib");
	261
	262	/*
	263	* IOC Aperture size decoded as 2 ^ (SIZE + 2) KB,
	264	* so setting 0x11 implies 512M, 0x12 implies 1G...
	265	*/
	266	write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE,
	267	order_base_2(ap_size/1024) - 2);
	268
	269
	270	/* IOC Aperture start must be aligned to the size of the aperture */
	271	if (ap_base % ap_size != 0)
	272	panic("IOC Aperture start must be aligned to the size of the aperture");
	273
	274	write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12);
db6ce231	275	write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1);
db6ce231	276	write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1);
97a63144	277
db6ce231	278	}
41cada4d EP	279
	280	read_decode_mmu_bcr();
	281
	282	/*
	283	* ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist
	284	* only if PAE exists in current HW. So we had to check pae_exist
	285	* before using them.
	286	*/
	287	if (slc_exists && pae_exists)
	288	slc_upper_region_init();
	289	#endif /* CONFIG_ISA_ARCV2 */
ef639e6f AB	290	}
ef639e6f AB	291
2f16ac9d AB	292	int icache_status(void)
2f16ac9d AB	293	{
379b3280	294	if (!icache_exists)
f8cf3d1e IG	295	return 0;
f8cf3d1e IG	296
ef639e6f AB	297	if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE)
	298	return 0;
	299	else
	300	return 1;
2f16ac9d AB	301	}
	302
	303	void icache_enable(void)
	304	{
379b3280	305	if (icache_exists)
ef639e6f AB	306	write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) &
ef639e6f AB	307	~IC_CTRL_CACHE_DISABLE);
2f16ac9d AB	308	}
	309
	310	void icache_disable(void)
	311	{
379b3280	312	if (icache_exists)
ef639e6f AB	313	write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) \|
ef639e6f AB	314	IC_CTRL_CACHE_DISABLE);
2f16ac9d AB	315	}
	316
	317	void invalidate_icache_all(void)
	318	{
2f16ac9d	319	/* Any write to IC_IVIC register triggers invalidation of entire I$ */
ef639e6f AB	320	if (icache_status()) {
ef639e6f AB	321	write_aux_reg(ARC_AUX_IC_IVIC, 1);
f2a22678 AB	322	/*
	323	* As per ARC HS databook (see chapter 5.3.3.2)
	324	* it is required to add 3 NOPs after each write to IC_IVIC.
	325	*/
	326	__builtin_arc_nop();
	327	__builtin_arc_nop();
	328	__builtin_arc_nop();
ef639e6f AB	329	read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */
ef639e6f AB	330	}
41cada4d EP	331
	332	#ifdef CONFIG_ISA_ARCV2
	333	if (slc_exists)
	334	__slc_entire_op(OP_INV);
ef639e6f	335	#endif
41cada4d	336	}
2f16ac9d AB	337
	338	int dcache_status(void)
	339	{
379b3280	340	if (!dcache_exists)
f8cf3d1e IG	341	return 0;
f8cf3d1e IG	342
ef639e6f AB	343	if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE)
	344	return 0;
	345	else
	346	return 1;
2f16ac9d AB	347	}
	348
	349	void dcache_enable(void)
	350	{
379b3280	351	if (!dcache_exists)
f8cf3d1e IG	352	return;
f8cf3d1e IG	353
2f16ac9d AB	354	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) &
	355	~(DC_CTRL_INV_MODE_FLUSH \| DC_CTRL_CACHE_DISABLE));
	356	}
	357
	358	void dcache_disable(void)
	359	{
379b3280	360	if (!dcache_exists)
f8cf3d1e IG	361	return;
f8cf3d1e IG	362
2f16ac9d AB	363	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) \|
	364	DC_CTRL_CACHE_DISABLE);
	365	}
	366
2f16ac9d	367	#ifndef CONFIG_SYS_DCACHE_OFF
ef639e6f AB	368	/*
	369	* Common Helper for Line Operations on {I,D}-Cache
	370	*/
	371	static inline void __cache_line_loop(unsigned long paddr, unsigned long sz,
	372	const int cacheop)
2f16ac9d	373	{
ef639e6f	374	unsigned int aux_cmd;
5ff40f3d	375	#if (CONFIG_ARC_MMU_VER == 3)
ef639e6f	376	unsigned int aux_tag;
2f16ac9d	377	#endif
ef639e6f	378	int num_lines;
2f16ac9d	379
ef639e6f AB	380	if (cacheop == OP_INV_IC) {
ef639e6f AB	381	aux_cmd = ARC_AUX_IC_IVIL;
5ff40f3d	382	#if (CONFIG_ARC_MMU_VER == 3)
ef639e6f	383	aux_tag = ARC_AUX_IC_PTAG;
2f16ac9d	384	#endif
ef639e6f AB	385	} else {
	386	/* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
	387	aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL;
	388	#if (CONFIG_ARC_MMU_VER == 3)
	389	aux_tag = ARC_AUX_DC_PTAG;
	390	#endif
	391	}
2f16ac9d	392
ef639e6f AB	393	sz += paddr & ~CACHE_LINE_MASK;
ef639e6f AB	394	paddr &= CACHE_LINE_MASK;
2f16ac9d	395
379b3280	396	num_lines = DIV_ROUND_UP(sz, l1_line_sz);
2f16ac9d	397
ef639e6f	398	while (num_lines-- > 0) {
5ff40f3d	399	#if (CONFIG_ARC_MMU_VER == 3)
ef639e6f	400	write_aux_reg(aux_tag, paddr);
2f16ac9d	401	#endif
ef639e6f	402	write_aux_reg(aux_cmd, paddr);
379b3280	403	paddr += l1_line_sz;
2f16ac9d	404	}
2f16ac9d AB	405	}
2f16ac9d AB	406
ef639e6f	407	static unsigned int __before_dc_op(const int op)
2f16ac9d	408	{
ef639e6f AB	409	unsigned int reg;
	410
	411	if (op == OP_INV) {
	412	/*
	413	* IM is set by default and implies Flush-n-inv
	414	* Clear it here for vanilla inv
	415	*/
	416	reg = read_aux_reg(ARC_AUX_DC_CTRL);
	417	write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
	418	}
ae4a351a	419
ef639e6f	420	return reg;
2f16ac9d AB	421	}
2f16ac9d AB	422
ef639e6f	423	static void __after_dc_op(const int op, unsigned int reg)
2f16ac9d	424	{
ef639e6f AB	425	if (op & OP_FLUSH) /* flush / flush-n-inv both wait */
	426	while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
	427	;
	428
	429	/* Switch back to default Invalidate mode */
	430	if (op == OP_INV)
	431	write_aux_reg(ARC_AUX_DC_CTRL, reg \| DC_CTRL_INV_MODE_FLUSH);
2f16ac9d	432	}
6eb15e50	433
ef639e6f	434	static inline void __dc_entire_op(const int cacheop)
6eb15e50	435	{
ef639e6f AB	436	int aux;
ef639e6f AB	437	unsigned int ctrl_reg = __before_dc_op(cacheop);
6eb15e50	438
ef639e6f AB	439	if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */
	440	aux = ARC_AUX_DC_IVDC;
	441	else
	442	aux = ARC_AUX_DC_FLSH;
6eb15e50	443
ef639e6f	444	write_aux_reg(aux, 0x1);
6eb15e50	445
ef639e6f	446	__after_dc_op(cacheop, ctrl_reg);
6eb15e50 AB	447	}
6eb15e50 AB	448
ef639e6f AB	449	static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
ef639e6f AB	450	const int cacheop)
6eb15e50	451	{
ef639e6f AB	452	unsigned int ctrl_reg = __before_dc_op(cacheop);
	453	__cache_line_loop(paddr, sz, cacheop);
	454	__after_dc_op(cacheop, ctrl_reg);
	455	}
	456	#else
	457	#define __dc_entire_op(cacheop)
	458	#define __dc_line_op(paddr, sz, cacheop)
	459	#endif /* !CONFIG_SYS_DCACHE_OFF */
6eb15e50	460
ef639e6f AB	461	void invalidate_dcache_range(unsigned long start, unsigned long end)
ef639e6f AB	462	{
41cada4d EP	463	if (start >= end)
	464	return;
	465
ef639e6f	466	#ifdef CONFIG_ISA_ARCV2
db6ce231 AB	467	if (!ioc_exists)
	468	#endif
	469	__dc_line_op(start, end - start, OP_INV);
	470
	471	#ifdef CONFIG_ISA_ARCV2
	472	if (slc_exists && !ioc_exists)
41cada4d	473	__slc_rgn_op(start, end - start, OP_INV);
ef639e6f AB	474	#endif
ef639e6f AB	475	}
6eb15e50	476
ef639e6f AB	477	void flush_dcache_range(unsigned long start, unsigned long end)
ef639e6f AB	478	{
41cada4d EP	479	if (start >= end)
	480	return;
	481
ef639e6f	482	#ifdef CONFIG_ISA_ARCV2
db6ce231 AB	483	if (!ioc_exists)
	484	#endif
	485	__dc_line_op(start, end - start, OP_FLUSH);
	486
	487	#ifdef CONFIG_ISA_ARCV2
	488	if (slc_exists && !ioc_exists)
41cada4d	489	__slc_rgn_op(start, end - start, OP_FLUSH);
ef639e6f	490	#endif
6eb15e50 AB	491	}
6eb15e50 AB	492
ef639e6f	493	void flush_cache(unsigned long start, unsigned long size)
6eb15e50	494	{
ef639e6f AB	495	flush_dcache_range(start, start + size);
ef639e6f AB	496	}
6eb15e50	497
ef639e6f AB	498	void invalidate_dcache_all(void)
ef639e6f AB	499	{
bd91508b	500	__dc_entire_op(OP_INV);
db6ce231 AB	501
db6ce231 AB	502	#ifdef CONFIG_ISA_ARCV2
bd91508b	503	if (slc_exists)
ef639e6f AB	504	__slc_entire_op(OP_INV);
ef639e6f AB	505	#endif
6eb15e50 AB	506	}
6eb15e50 AB	507
ef639e6f AB	508	void flush_dcache_all(void)
ef639e6f AB	509	{
2a8382c6	510	__dc_entire_op(OP_FLUSH);
db6ce231 AB	511
db6ce231 AB	512	#ifdef CONFIG_ISA_ARCV2
2a8382c6	513	if (slc_exists)
ef639e6f AB	514	__slc_entire_op(OP_FLUSH);
	515	#endif
	516	}