[people/ms/u-boot.git] / arch / arc / lib / cache.c

/*
 * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved.
 *
 * SPDX-License-Identifier:	GPL-2.0+
 */

#include <config.h>
#include <common.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <asm/arcregs.h>
#include <asm/cache.h>

/* Bit values in IC_CTRL */
#define IC_CTRL_CACHE_DISABLE	(1 << 0)

/* Bit values in DC_CTRL */
#define DC_CTRL_CACHE_DISABLE	(1 << 0)
#define DC_CTRL_INV_MODE_FLUSH	(1 << 6)
#define DC_CTRL_FLUSH_STATUS	(1 << 8)
#define CACHE_VER_NUM_MASK	0xF
#define SLC_CTRL_SB		(1 << 2)

#define OP_INV		0x1
#define OP_FLUSH	0x2
#define OP_INV_IC	0x3

/*
 * By default that variable will fall into .bss section.
 * But .bss section is not relocated and so it will be initilized before
 * relocation but will be used after being zeroed.
 */
int l1_line_sz __section(".data");
int dcache_exists __section(".data");
int icache_exists __section(".data");

#define CACHE_LINE_MASK		(~(l1_line_sz - 1))

#ifdef CONFIG_ISA_ARCV2
int slc_line_sz __section(".data");
int slc_exists __section(".data");
int ioc_exists __section(".data");

static unsigned int __before_slc_op(const int op)
{
	unsigned int reg = reg;

	if (op == OP_INV) {
		/*
		 * IM is set by default and implies Flush-n-inv
		 * Clear it here for vanilla inv
		 */
		reg = read_aux_reg(ARC_AUX_SLC_CTRL);
		write_aux_reg(ARC_AUX_SLC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
	}

	return reg;
}

static void __after_slc_op(const int op, unsigned int reg)
{
	if (op & OP_FLUSH) {	/* flush / flush-n-inv both wait */
		/*
		 * Make sure "busy" bit reports correct status,
		 * see STAR 9001165532
		 */
		read_aux_reg(ARC_AUX_SLC_CTRL);
		while (read_aux_reg(ARC_AUX_SLC_CTRL) &
		       DC_CTRL_FLUSH_STATUS)
			;
	}

	/* Switch back to default Invalidate mode */
	if (op == OP_INV)
		write_aux_reg(ARC_AUX_SLC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH);
}

static inline void __slc_line_loop(unsigned long paddr, unsigned long sz,
				   const int op)
{
	unsigned int aux_cmd;
	int num_lines;

#define SLC_LINE_MASK	(~(slc_line_sz - 1))

	aux_cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL;

	sz += paddr & ~SLC_LINE_MASK;
	paddr &= SLC_LINE_MASK;

	num_lines = DIV_ROUND_UP(sz, slc_line_sz);

	while (num_lines-- > 0) {
		write_aux_reg(aux_cmd, paddr);
		paddr += slc_line_sz;
	}
}

static inline void __slc_entire_op(const int cacheop)
{
	int aux;
	unsigned int ctrl_reg = __before_slc_op(cacheop);

	if (cacheop & OP_INV)	/* Inv or flush-n-inv use same cmd reg */
		aux = ARC_AUX_SLC_INVALIDATE;
	else
		aux = ARC_AUX_SLC_FLUSH;

	write_aux_reg(aux, 0x1);

	__after_slc_op(cacheop, ctrl_reg);
}

static inline void __slc_line_op(unsigned long paddr, unsigned long sz,
				 const int cacheop)
{
	unsigned int ctrl_reg = __before_slc_op(cacheop);
	__slc_line_loop(paddr, sz, cacheop);
	__after_slc_op(cacheop, ctrl_reg);
}
#else
#define __slc_entire_op(cacheop)
#define __slc_line_op(paddr, sz, cacheop)
#endif

#ifdef CONFIG_ISA_ARCV2
static void read_decode_cache_bcr_arcv2(void)
{
	union {
		struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:24, way:2, lsz:2, sz:4;
#else
			unsigned int sz:4, lsz:2, way:2, pad:24;
#endif
		} fields;
		unsigned int word;
	} slc_cfg;

	union {
		struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:24, ver:8;
#else
			unsigned int ver:8, pad:24;
#endif
		} fields;
		unsigned int word;
	} sbcr;

	sbcr.word = read_aux_reg(ARC_BCR_SLC);
	if (sbcr.fields.ver) {
		slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG);
		slc_exists = 1;
		slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64;
	}

	union {
		struct bcr_clust_cfg {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8;
#else
			unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7;
#endif
		} fields;
		unsigned int word;
	} cbcr;

	cbcr.word = read_aux_reg(ARC_BCR_CLUSTER);
	if (cbcr.fields.c)
		ioc_exists = 1;
}
#endif

void read_decode_cache_bcr(void)
{
	int dc_line_sz = 0, ic_line_sz = 0;

	union {
		struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
			unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
#else
			unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
#endif
		} fields;
		unsigned int word;
	} ibcr, dbcr;

	ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
	if (ibcr.fields.ver) {
		icache_exists = 1;
		l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len;
		if (!ic_line_sz)
			panic("Instruction exists but line length is 0\n");
	}

	dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
	if (dbcr.fields.ver){
		dcache_exists = 1;
		l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len;
		if (!dc_line_sz)
			panic("Data cache exists but line length is 0\n");
	}

	if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz))
		panic("Instruction and data cache line lengths differ\n");
}

void cache_init(void)
{
	read_decode_cache_bcr();

#ifdef CONFIG_ISA_ARCV2
	read_decode_cache_bcr_arcv2();

	if (ioc_exists) {
		flush_dcache_all();
		invalidate_dcache_all();

		/* IO coherency base - 0x8z */
		write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, 0x80000);
		/* IO coherency aperture size - 512Mb: 0x8z-0xAz */
		write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 0x11);
		/* Enable partial writes */
		write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1);
		/* Enable IO coherency */
		write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1);
	}
#endif
}

int icache_status(void)
{
	if (!icache_exists)
		return 0;

	if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE)
		return 0;
	else
		return 1;
}

void icache_enable(void)
{
	if (icache_exists)
		write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) &
			      ~IC_CTRL_CACHE_DISABLE);
}

void icache_disable(void)
{
	if (icache_exists)
		write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) |
			      IC_CTRL_CACHE_DISABLE);
}

#ifndef CONFIG_SYS_DCACHE_OFF
void invalidate_icache_all(void)
{
	/* Any write to IC_IVIC register triggers invalidation of entire I$ */
	if (icache_status()) {
		write_aux_reg(ARC_AUX_IC_IVIC, 1);
		read_aux_reg(ARC_AUX_IC_CTRL);	/* blocks */
	}
}
#else
void invalidate_icache_all(void)
{
}
#endif

int dcache_status(void)
{
	if (!dcache_exists)
		return 0;

	if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE)
		return 0;
	else
		return 1;
}

void dcache_enable(void)
{
	if (!dcache_exists)
		return;

	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) &
		      ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE));
}

void dcache_disable(void)
{
	if (!dcache_exists)
		return;

	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) |
		      DC_CTRL_CACHE_DISABLE);
}

#ifndef CONFIG_SYS_DCACHE_OFF
/*
 * Common Helper for Line Operations on {I,D}-Cache
 */
static inline void __cache_line_loop(unsigned long paddr, unsigned long sz,
				     const int cacheop)
{
	unsigned int aux_cmd;
#if (CONFIG_ARC_MMU_VER == 3)
	unsigned int aux_tag;
#endif
	int num_lines;

	if (cacheop == OP_INV_IC) {
		aux_cmd = ARC_AUX_IC_IVIL;
#if (CONFIG_ARC_MMU_VER == 3)
		aux_tag = ARC_AUX_IC_PTAG;
#endif
	} else {
		/* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
		aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL;
#if (CONFIG_ARC_MMU_VER == 3)
		aux_tag = ARC_AUX_DC_PTAG;
#endif
	}

	sz += paddr & ~CACHE_LINE_MASK;
	paddr &= CACHE_LINE_MASK;

	num_lines = DIV_ROUND_UP(sz, l1_line_sz);

	while (num_lines-- > 0) {
#if (CONFIG_ARC_MMU_VER == 3)
		write_aux_reg(aux_tag, paddr);
#endif
		write_aux_reg(aux_cmd, paddr);
		paddr += l1_line_sz;
	}
}

static unsigned int __before_dc_op(const int op)
{
	unsigned int reg;

	if (op == OP_INV) {
		/*
		 * IM is set by default and implies Flush-n-inv
		 * Clear it here for vanilla inv
		 */
		reg = read_aux_reg(ARC_AUX_DC_CTRL);
		write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
	}

	return reg;
}

static void __after_dc_op(const int op, unsigned int reg)
{
	if (op & OP_FLUSH)	/* flush / flush-n-inv both wait */
		while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
			;

	/* Switch back to default Invalidate mode */
	if (op == OP_INV)
		write_aux_reg(ARC_AUX_DC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH);
}

static inline void __dc_entire_op(const int cacheop)
{
	int aux;
	unsigned int ctrl_reg = __before_dc_op(cacheop);

	if (cacheop & OP_INV)	/* Inv or flush-n-inv use same cmd reg */
		aux = ARC_AUX_DC_IVDC;
	else
		aux = ARC_AUX_DC_FLSH;

	write_aux_reg(aux, 0x1);

	__after_dc_op(cacheop, ctrl_reg);
}

static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
				const int cacheop)
{
	unsigned int ctrl_reg = __before_dc_op(cacheop);
	__cache_line_loop(paddr, sz, cacheop);
	__after_dc_op(cacheop, ctrl_reg);
}
#else
#define __dc_entire_op(cacheop)
#define __dc_line_op(paddr, sz, cacheop)
#endif /* !CONFIG_SYS_DCACHE_OFF */

void invalidate_dcache_range(unsigned long start, unsigned long end)
{
#ifdef CONFIG_ISA_ARCV2
	if (!ioc_exists)
#endif
		__dc_line_op(start, end - start, OP_INV);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists && !ioc_exists)
		__slc_line_op(start, end - start, OP_INV);
#endif
}

void flush_dcache_range(unsigned long start, unsigned long end)
{
#ifdef CONFIG_ISA_ARCV2
	if (!ioc_exists)
#endif
		__dc_line_op(start, end - start, OP_FLUSH);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists && !ioc_exists)
		__slc_line_op(start, end - start, OP_FLUSH);
#endif
}

void flush_cache(unsigned long start, unsigned long size)
{
	flush_dcache_range(start, start + size);
}

void invalidate_dcache_all(void)
{
	__dc_entire_op(OP_INV);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists)
		__slc_entire_op(OP_INV);
#endif
}

void flush_dcache_all(void)
{
	__dc_entire_op(OP_FLUSH);

#ifdef CONFIG_ISA_ARCV2
	if (slc_exists)
		__slc_entire_op(OP_FLUSH);
#endif
}
Commit	Line	Data
2f16ac9d AB	1	/*
	2	* Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved.
	3	*
	4	* SPDX-License-Identifier: GPL-2.0+
	5	*/
	6
	7	#include <config.h>
379b3280	8	#include <common.h>
ef639e6f AB	9	#include <linux/compiler.h>
ef639e6f AB	10	#include <linux/kernel.h>
2f16ac9d	11	#include <asm/arcregs.h>
205e7a7b	12	#include <asm/cache.h>
2f16ac9d AB	13
	14	/* Bit values in IC_CTRL */
	15	#define IC_CTRL_CACHE_DISABLE (1 << 0)
	16
	17	/* Bit values in DC_CTRL */
	18	#define DC_CTRL_CACHE_DISABLE (1 << 0)
	19	#define DC_CTRL_INV_MODE_FLUSH (1 << 6)
	20	#define DC_CTRL_FLUSH_STATUS (1 << 8)
f8cf3d1e	21	#define CACHE_VER_NUM_MASK 0xF
6eb15e50	22	#define SLC_CTRL_SB (1 << 2)
2f16ac9d	23
ef639e6f AB	24	#define OP_INV 0x1
	25	#define OP_FLUSH 0x2
	26	#define OP_INV_IC 0x3
	27
ef639e6f AB	28	/*
	29	* By default that variable will fall into .bss section.
	30	* But .bss section is not relocated and so it will be initilized before
	31	* relocation but will be used after being zeroed.
	32	*/
379b3280 AB	33	int l1_line_sz __section(".data");
	34	int dcache_exists __section(".data");
	35	int icache_exists __section(".data");
	36
	37	#define CACHE_LINE_MASK (~(l1_line_sz - 1))
	38
	39	#ifdef CONFIG_ISA_ARCV2
ef639e6f AB	40	int slc_line_sz __section(".data");
ef639e6f AB	41	int slc_exists __section(".data");
db6ce231	42	int ioc_exists __section(".data");
ef639e6f AB	43
	44	static unsigned int __before_slc_op(const int op)
	45	{
	46	unsigned int reg = reg;
	47
	48	if (op == OP_INV) {
	49	/*
	50	* IM is set by default and implies Flush-n-inv
	51	* Clear it here for vanilla inv
	52	*/
	53	reg = read_aux_reg(ARC_AUX_SLC_CTRL);
	54	write_aux_reg(ARC_AUX_SLC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
	55	}
	56
	57	return reg;
	58	}
	59
	60	static void __after_slc_op(const int op, unsigned int reg)
	61	{
40a808f1 AB	62	if (op & OP_FLUSH) { /* flush / flush-n-inv both wait */
	63	/*
	64	* Make sure "busy" bit reports correct status,
	65	* see STAR 9001165532
	66	*/
	67	read_aux_reg(ARC_AUX_SLC_CTRL);
ef639e6f AB	68	while (read_aux_reg(ARC_AUX_SLC_CTRL) &
	69	DC_CTRL_FLUSH_STATUS)
	70	;
40a808f1	71	}
ef639e6f AB	72
	73	/* Switch back to default Invalidate mode */
	74	if (op == OP_INV)
	75	write_aux_reg(ARC_AUX_SLC_CTRL, reg \| DC_CTRL_INV_MODE_FLUSH);
	76	}
	77
	78	static inline void __slc_line_loop(unsigned long paddr, unsigned long sz,
	79	const int op)
	80	{
	81	unsigned int aux_cmd;
	82	int num_lines;
	83
	84	#define SLC_LINE_MASK (~(slc_line_sz - 1))
	85
	86	aux_cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL;
	87
	88	sz += paddr & ~SLC_LINE_MASK;
	89	paddr &= SLC_LINE_MASK;
	90
	91	num_lines = DIV_ROUND_UP(sz, slc_line_sz);
	92
	93	while (num_lines-- > 0) {
	94	write_aux_reg(aux_cmd, paddr);
	95	paddr += slc_line_sz;
	96	}
	97	}
	98
	99	static inline void __slc_entire_op(const int cacheop)
	100	{
	101	int aux;
	102	unsigned int ctrl_reg = __before_slc_op(cacheop);
	103
	104	if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */
	105	aux = ARC_AUX_SLC_INVALIDATE;
	106	else
	107	aux = ARC_AUX_SLC_FLUSH;
	108
	109	write_aux_reg(aux, 0x1);
	110
	111	__after_slc_op(cacheop, ctrl_reg);
	112	}
	113
	114	static inline void __slc_line_op(unsigned long paddr, unsigned long sz,
	115	const int cacheop)
	116	{
	117	unsigned int ctrl_reg = __before_slc_op(cacheop);
	118	__slc_line_loop(paddr, sz, cacheop);
	119	__after_slc_op(cacheop, ctrl_reg);
	120	}
	121	#else
	122	#define __slc_entire_op(cacheop)
	123	#define __slc_line_op(paddr, sz, cacheop)
	124	#endif
	125
379b3280 AB	126	#ifdef CONFIG_ISA_ARCV2
379b3280 AB	127	static void read_decode_cache_bcr_arcv2(void)
ef639e6f	128	{
379b3280 AB	129	union {
	130	struct {
	131	#ifdef CONFIG_CPU_BIG_ENDIAN
	132	unsigned int pad:24, way:2, lsz:2, sz:4;
	133	#else
	134	unsigned int sz:4, lsz:2, way:2, pad:24;
	135	#endif
	136	} fields;
	137	unsigned int word;
	138	} slc_cfg;
	139
	140	union {
	141	struct {
	142	#ifdef CONFIG_CPU_BIG_ENDIAN
	143	unsigned int pad:24, ver:8;
	144	#else
	145	unsigned int ver:8, pad:24;
	146	#endif
	147	} fields;
	148	unsigned int word;
	149	} sbcr;
	150
	151	sbcr.word = read_aux_reg(ARC_BCR_SLC);
	152	if (sbcr.fields.ver) {
	153	slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG);
	154	slc_exists = 1;
	155	slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64;
	156	}
db6ce231 AB	157
	158	union {
	159	struct bcr_clust_cfg {
	160	#ifdef CONFIG_CPU_BIG_ENDIAN
	161	unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8;
	162	#else
	163	unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7;
	164	#endif
	165	} fields;
	166	unsigned int word;
	167	} cbcr;
	168
	169	cbcr.word = read_aux_reg(ARC_BCR_CLUSTER);
	170	if (cbcr.fields.c)
	171	ioc_exists = 1;
ef639e6f	172	}
379b3280	173	#endif
ef639e6f	174
379b3280	175	void read_decode_cache_bcr(void)
ef639e6f	176	{
379b3280 AB	177	int dc_line_sz = 0, ic_line_sz = 0;
	178
	179	union {
	180	struct {
	181	#ifdef CONFIG_CPU_BIG_ENDIAN
	182	unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
	183	#else
	184	unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
	185	#endif
	186	} fields;
	187	unsigned int word;
	188	} ibcr, dbcr;
	189
	190	ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
	191	if (ibcr.fields.ver) {
	192	icache_exists = 1;
	193	l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len;
	194	if (!ic_line_sz)
	195	panic("Instruction exists but line length is 0\n");
	196	}
	197
	198	dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
	199	if (dbcr.fields.ver){
	200	dcache_exists = 1;
	201	l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len;
	202	if (!dc_line_sz)
	203	panic("Data cache exists but line length is 0\n");
	204	}
	205
	206	if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz))
	207	panic("Instruction and data cache line lengths differ\n");
ef639e6f AB	208	}
	209
	210	void cache_init(void)
	211	{
379b3280 AB	212	read_decode_cache_bcr();
379b3280 AB	213
ef639e6f	214	#ifdef CONFIG_ISA_ARCV2
379b3280	215	read_decode_cache_bcr_arcv2();
db6ce231 AB	216
db6ce231 AB	217	if (ioc_exists) {
a4a43fcf AB	218	flush_dcache_all();
	219	invalidate_dcache_all();
	220
db6ce231 AB	221	/* IO coherency base - 0x8z */
	222	write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, 0x80000);
	223	/* IO coherency aperture size - 512Mb: 0x8z-0xAz */
	224	write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 0x11);
	225	/* Enable partial writes */
	226	write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1);
	227	/* Enable IO coherency */
	228	write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1);
	229	}
ef639e6f AB	230	#endif
	231	}
	232
2f16ac9d AB	233	int icache_status(void)
2f16ac9d AB	234	{
379b3280	235	if (!icache_exists)
f8cf3d1e IG	236	return 0;
f8cf3d1e IG	237
ef639e6f AB	238	if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE)
	239	return 0;
	240	else
	241	return 1;
2f16ac9d AB	242	}
	243
	244	void icache_enable(void)
	245	{
379b3280	246	if (icache_exists)
ef639e6f AB	247	write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) &
ef639e6f AB	248	~IC_CTRL_CACHE_DISABLE);
2f16ac9d AB	249	}
	250
	251	void icache_disable(void)
	252	{
379b3280	253	if (icache_exists)
ef639e6f AB	254	write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) \|
ef639e6f AB	255	IC_CTRL_CACHE_DISABLE);
2f16ac9d AB	256	}
2f16ac9d AB	257
ef639e6f	258	#ifndef CONFIG_SYS_DCACHE_OFF
2f16ac9d AB	259	void invalidate_icache_all(void)
2f16ac9d AB	260	{
2f16ac9d	261	/* Any write to IC_IVIC register triggers invalidation of entire I$ */
ef639e6f AB	262	if (icache_status()) {
	263	write_aux_reg(ARC_AUX_IC_IVIC, 1);
	264	read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */
	265	}
2f16ac9d	266	}
ef639e6f AB	267	#else
	268	void invalidate_icache_all(void)
	269	{
	270	}
	271	#endif
2f16ac9d AB	272
	273	int dcache_status(void)
	274	{
379b3280	275	if (!dcache_exists)
f8cf3d1e IG	276	return 0;
f8cf3d1e IG	277
ef639e6f AB	278	if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE)
	279	return 0;
	280	else
	281	return 1;
2f16ac9d AB	282	}
	283
	284	void dcache_enable(void)
	285	{
379b3280	286	if (!dcache_exists)
f8cf3d1e IG	287	return;
f8cf3d1e IG	288
2f16ac9d AB	289	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) &
	290	~(DC_CTRL_INV_MODE_FLUSH \| DC_CTRL_CACHE_DISABLE));
	291	}
	292
	293	void dcache_disable(void)
	294	{
379b3280	295	if (!dcache_exists)
f8cf3d1e IG	296	return;
f8cf3d1e IG	297
2f16ac9d AB	298	write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) \|
	299	DC_CTRL_CACHE_DISABLE);
	300	}
	301
2f16ac9d	302	#ifndef CONFIG_SYS_DCACHE_OFF
ef639e6f AB	303	/*
	304	* Common Helper for Line Operations on {I,D}-Cache
	305	*/
	306	static inline void __cache_line_loop(unsigned long paddr, unsigned long sz,
	307	const int cacheop)
2f16ac9d	308	{
ef639e6f	309	unsigned int aux_cmd;
5ff40f3d	310	#if (CONFIG_ARC_MMU_VER == 3)
ef639e6f	311	unsigned int aux_tag;
2f16ac9d	312	#endif
ef639e6f	313	int num_lines;
2f16ac9d	314
ef639e6f AB	315	if (cacheop == OP_INV_IC) {
ef639e6f AB	316	aux_cmd = ARC_AUX_IC_IVIL;
5ff40f3d	317	#if (CONFIG_ARC_MMU_VER == 3)
ef639e6f	318	aux_tag = ARC_AUX_IC_PTAG;
2f16ac9d	319	#endif
ef639e6f AB	320	} else {
	321	/* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
	322	aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL;
	323	#if (CONFIG_ARC_MMU_VER == 3)
	324	aux_tag = ARC_AUX_DC_PTAG;
	325	#endif
	326	}
2f16ac9d	327
ef639e6f AB	328	sz += paddr & ~CACHE_LINE_MASK;
ef639e6f AB	329	paddr &= CACHE_LINE_MASK;
2f16ac9d	330
379b3280	331	num_lines = DIV_ROUND_UP(sz, l1_line_sz);
2f16ac9d	332
ef639e6f	333	while (num_lines-- > 0) {
5ff40f3d	334	#if (CONFIG_ARC_MMU_VER == 3)
ef639e6f	335	write_aux_reg(aux_tag, paddr);
2f16ac9d	336	#endif
ef639e6f	337	write_aux_reg(aux_cmd, paddr);
379b3280	338	paddr += l1_line_sz;
2f16ac9d	339	}
2f16ac9d AB	340	}
2f16ac9d AB	341
ef639e6f	342	static unsigned int __before_dc_op(const int op)
2f16ac9d	343	{
ef639e6f AB	344	unsigned int reg;
	345
	346	if (op == OP_INV) {
	347	/*
	348	* IM is set by default and implies Flush-n-inv
	349	* Clear it here for vanilla inv
	350	*/
	351	reg = read_aux_reg(ARC_AUX_DC_CTRL);
	352	write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
	353	}
ae4a351a	354
ef639e6f	355	return reg;
2f16ac9d AB	356	}
2f16ac9d AB	357
ef639e6f	358	static void __after_dc_op(const int op, unsigned int reg)
2f16ac9d	359	{
ef639e6f AB	360	if (op & OP_FLUSH) /* flush / flush-n-inv both wait */
	361	while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
	362	;
	363
	364	/* Switch back to default Invalidate mode */
	365	if (op == OP_INV)
	366	write_aux_reg(ARC_AUX_DC_CTRL, reg \| DC_CTRL_INV_MODE_FLUSH);
2f16ac9d	367	}
6eb15e50	368
ef639e6f	369	static inline void __dc_entire_op(const int cacheop)
6eb15e50	370	{
ef639e6f AB	371	int aux;
ef639e6f AB	372	unsigned int ctrl_reg = __before_dc_op(cacheop);
6eb15e50	373
ef639e6f AB	374	if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */
	375	aux = ARC_AUX_DC_IVDC;
	376	else
	377	aux = ARC_AUX_DC_FLSH;
6eb15e50	378
ef639e6f	379	write_aux_reg(aux, 0x1);
6eb15e50	380
ef639e6f	381	__after_dc_op(cacheop, ctrl_reg);
6eb15e50 AB	382	}
6eb15e50 AB	383
ef639e6f AB	384	static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
ef639e6f AB	385	const int cacheop)
6eb15e50	386	{
ef639e6f AB	387	unsigned int ctrl_reg = __before_dc_op(cacheop);
	388	__cache_line_loop(paddr, sz, cacheop);
	389	__after_dc_op(cacheop, ctrl_reg);
	390	}
	391	#else
	392	#define __dc_entire_op(cacheop)
	393	#define __dc_line_op(paddr, sz, cacheop)
	394	#endif /* !CONFIG_SYS_DCACHE_OFF */
6eb15e50	395
ef639e6f AB	396	void invalidate_dcache_range(unsigned long start, unsigned long end)
ef639e6f AB	397	{
ef639e6f	398	#ifdef CONFIG_ISA_ARCV2
db6ce231 AB	399	if (!ioc_exists)
	400	#endif
	401	__dc_line_op(start, end - start, OP_INV);
	402
	403	#ifdef CONFIG_ISA_ARCV2
	404	if (slc_exists && !ioc_exists)
ef639e6f AB	405	__slc_line_op(start, end - start, OP_INV);
	406	#endif
	407	}
6eb15e50	408
ef639e6f AB	409	void flush_dcache_range(unsigned long start, unsigned long end)
ef639e6f AB	410	{
ef639e6f	411	#ifdef CONFIG_ISA_ARCV2
db6ce231 AB	412	if (!ioc_exists)
	413	#endif
	414	__dc_line_op(start, end - start, OP_FLUSH);
	415
	416	#ifdef CONFIG_ISA_ARCV2
	417	if (slc_exists && !ioc_exists)
ef639e6f AB	418	__slc_line_op(start, end - start, OP_FLUSH);
ef639e6f AB	419	#endif
6eb15e50 AB	420	}
6eb15e50 AB	421
ef639e6f	422	void flush_cache(unsigned long start, unsigned long size)
6eb15e50	423	{
ef639e6f AB	424	flush_dcache_range(start, start + size);
ef639e6f AB	425	}
6eb15e50	426
ef639e6f AB	427	void invalidate_dcache_all(void)
ef639e6f AB	428	{
bd91508b	429	__dc_entire_op(OP_INV);
db6ce231 AB	430
db6ce231 AB	431	#ifdef CONFIG_ISA_ARCV2
bd91508b	432	if (slc_exists)
ef639e6f AB	433	__slc_entire_op(OP_INV);
ef639e6f AB	434	#endif
6eb15e50 AB	435	}
6eb15e50 AB	436
ef639e6f AB	437	void flush_dcache_all(void)
ef639e6f AB	438	{
2a8382c6	439	__dc_entire_op(OP_FLUSH);
db6ce231 AB	440
db6ce231 AB	441	#ifdef CONFIG_ISA_ARCV2
2a8382c6	442	if (slc_exists)
ef639e6f AB	443	__slc_entire_op(OP_FLUSH);
	444	#endif
	445	}