]>
Commit | Line | Data |
---|---|---|
83d290c5 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
2f16ac9d AB |
2 | /* |
3 | * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. | |
2f16ac9d AB |
4 | */ |
5 | ||
6 | #include <config.h> | |
9edefc27 | 7 | #include <cpu_func.h> |
401d1c4f | 8 | #include <asm/global_data.h> |
cd93d625 | 9 | #include <linux/bitops.h> |
ef639e6f AB |
10 | #include <linux/compiler.h> |
11 | #include <linux/kernel.h> | |
97a63144 | 12 | #include <linux/log2.h> |
cfa19719 | 13 | #include <lmb.h> |
2f16ac9d | 14 | #include <asm/arcregs.h> |
88ae27ed | 15 | #include <asm/arc-bcr.h> |
205e7a7b | 16 | #include <asm/cache.h> |
2f16ac9d | 17 | |
c27814be EP |
18 | /* |
19 | * [ NOTE 1 ]: | |
20 | * Data cache (L1 D$ or SL$) entire invalidate operation or data cache disable | |
21 | * operation may result in unexpected behavior and data loss even if we flush | |
22 | * data cache right before invalidation. That may happens if we store any context | |
23 | * on stack (like we store BLINK register on stack before function call). | |
24 | * BLINK register is the register where return address is automatically saved | |
25 | * when we do function call with instructions like 'bl'. | |
26 | * | |
27 | * There is the real example: | |
28 | * We may hang in the next code as we store any BLINK register on stack in | |
29 | * invalidate_dcache_all() function. | |
30 | * | |
31 | * void flush_dcache_all() { | |
32 | * __dc_entire_op(OP_FLUSH); | |
33 | * // Other code // | |
34 | * } | |
35 | * | |
36 | * void invalidate_dcache_all() { | |
37 | * __dc_entire_op(OP_INV); | |
38 | * // Other code // | |
39 | * } | |
40 | * | |
41 | * void foo(void) { | |
42 | * flush_dcache_all(); | |
43 | * invalidate_dcache_all(); | |
44 | * } | |
45 | * | |
46 | * Now let's see what really happens during that code execution: | |
47 | * | |
48 | * foo() | |
49 | * |->> call flush_dcache_all | |
50 | * [return address is saved to BLINK register] | |
51 | * [push BLINK] (save to stack) ![point 1] | |
52 | * |->> call __dc_entire_op(OP_FLUSH) | |
53 | * [return address is saved to BLINK register] | |
54 | * [flush L1 D$] | |
55 | * return [jump to BLINK] | |
56 | * <<------ | |
57 | * [other flush_dcache_all code] | |
58 | * [pop BLINK] (get from stack) | |
59 | * return [jump to BLINK] | |
60 | * <<------ | |
61 | * |->> call invalidate_dcache_all | |
62 | * [return address is saved to BLINK register] | |
63 | * [push BLINK] (save to stack) ![point 2] | |
64 | * |->> call __dc_entire_op(OP_FLUSH) | |
65 | * [return address is saved to BLINK register] | |
66 | * [invalidate L1 D$] ![point 3] | |
67 | * // Oops!!! | |
68 | * // We lose return address from invalidate_dcache_all function: | |
69 | * // we save it to stack and invalidate L1 D$ after that! | |
70 | * return [jump to BLINK] | |
71 | * <<------ | |
72 | * [other invalidate_dcache_all code] | |
73 | * [pop BLINK] (get from stack) | |
74 | * // we don't have this data in L1 dcache as we invalidated it in [point 3] | |
75 | * // so we get it from next memory level (for example DDR memory) | |
76 | * // but in the memory we have value which we save in [point 1], which | |
77 | * // is return address from flush_dcache_all function (instead of | |
78 | * // address from current invalidate_dcache_all function which we | |
79 | * // saved in [point 2] !) | |
80 | * return [jump to BLINK] | |
81 | * <<------ | |
82 | * // As BLINK points to invalidate_dcache_all, we call it again and | |
83 | * // loop forever. | |
84 | * | |
85 | * Fortunately we may fix that by using flush & invalidation of D$ with a single | |
86 | * one instruction (instead of flush and invalidation instructions pair) and | |
87 | * enabling force function inline with '__attribute__((always_inline))' gcc | |
88 | * attribute to avoid any function call (and BLINK store) between cache flush | |
89 | * and disable. | |
7241944a EP |
90 | * |
91 | * | |
92 | * [ NOTE 2 ]: | |
93 | * As of today we only support the following cache configurations on ARC. | |
b15cb0bf | 94 | * Other configurations may exist in HW but we don't support it in SW. |
7241944a EP |
95 | * Configuration 1: |
96 | * ______________________ | |
97 | * | | | |
98 | * | ARC CPU | | |
99 | * |______________________| | |
100 | * ___|___ ___|___ | |
101 | * | | | | | |
102 | * | L1 I$ | | L1 D$ | | |
103 | * |_______| |_______| | |
104 | * on/off on/off | |
105 | * ___|______________|____ | |
106 | * | | | |
107 | * | main memory | | |
108 | * |______________________| | |
109 | * | |
110 | * Configuration 2: | |
111 | * ______________________ | |
112 | * | | | |
113 | * | ARC CPU | | |
114 | * |______________________| | |
115 | * ___|___ ___|___ | |
116 | * | | | | | |
117 | * | L1 I$ | | L1 D$ | | |
118 | * |_______| |_______| | |
119 | * on/off on/off | |
120 | * ___|______________|____ | |
121 | * | | | |
122 | * | L2 (SL$) | | |
123 | * |______________________| | |
b15cb0bf EP |
124 | * always on (ARCv2, HS < 3.0) |
125 | * on/off (ARCv2, HS >= 3.0) | |
7241944a EP |
126 | * ___|______________|____ |
127 | * | | | |
128 | * | main memory | | |
129 | * |______________________| | |
130 | * | |
131 | * Configuration 3: | |
132 | * ______________________ | |
133 | * | | | |
134 | * | ARC CPU | | |
135 | * |______________________| | |
136 | * ___|___ ___|___ | |
137 | * | | | | | |
138 | * | L1 I$ | | L1 D$ | | |
139 | * |_______| |_______| | |
140 | * on/off must be on | |
141 | * ___|______________|____ _______ | |
142 | * | | | | | |
143 | * | L2 (SL$) |-----| IOC | | |
144 | * |______________________| |_______| | |
145 | * always must be on on/off | |
146 | * ___|______________|____ | |
147 | * | | | |
148 | * | main memory | | |
149 | * |______________________| | |
c27814be EP |
150 | */ |
151 | ||
bf8974ed EP |
152 | DECLARE_GLOBAL_DATA_PTR; |
153 | ||
2f16ac9d | 154 | /* Bit values in IC_CTRL */ |
19b10a42 | 155 | #define IC_CTRL_CACHE_DISABLE BIT(0) |
2f16ac9d AB |
156 | |
157 | /* Bit values in DC_CTRL */ | |
19b10a42 EP |
158 | #define DC_CTRL_CACHE_DISABLE BIT(0) |
159 | #define DC_CTRL_INV_MODE_FLUSH BIT(6) | |
160 | #define DC_CTRL_FLUSH_STATUS BIT(8) | |
2f16ac9d | 161 | |
5d7a24d6 EP |
162 | #define OP_INV BIT(0) |
163 | #define OP_FLUSH BIT(1) | |
164 | #define OP_FLUSH_N_INV (OP_FLUSH | OP_INV) | |
ef639e6f | 165 | |
41cada4d EP |
166 | /* Bit val in SLC_CONTROL */ |
167 | #define SLC_CTRL_DIS 0x001 | |
168 | #define SLC_CTRL_IM 0x040 | |
169 | #define SLC_CTRL_BUSY 0x100 | |
170 | #define SLC_CTRL_RGN_OP_INV 0x200 | |
171 | ||
bf8974ed | 172 | #define CACHE_LINE_MASK (~(gd->arch.l1_line_sz - 1)) |
379b3280 | 173 | |
9f0253c6 EP |
174 | /* |
175 | * We don't want to use '__always_inline' macro here as it can be redefined | |
176 | * to simple 'inline' in some cases which breaks stuff. See [ NOTE 1 ] for more | |
177 | * details about the reasons we need to use always_inline functions. | |
178 | */ | |
179 | #define inlined_cachefunc inline __attribute__((always_inline)) | |
180 | ||
181 | static inlined_cachefunc void __ic_entire_invalidate(void); | |
182 | static inlined_cachefunc void __dc_entire_op(const int cacheop); | |
b15cb0bf | 183 | static inlined_cachefunc void __slc_entire_op(const int op); |
04286d07 | 184 | static inlined_cachefunc bool ioc_enabled(void); |
9f0253c6 | 185 | |
75790873 | 186 | static inline bool pae_exists(void) |
ef639e6f | 187 | { |
41cada4d EP |
188 | /* TODO: should we compare mmu version from BCR and from CONFIG? */ |
189 | #if (CONFIG_ARC_MMU_VER >= 4) | |
88ae27ed | 190 | union bcr_mmu_4 mmu4; |
ef639e6f | 191 | |
88ae27ed | 192 | mmu4.word = read_aux_reg(ARC_AUX_MMU_BCR); |
ef639e6f | 193 | |
75790873 EP |
194 | if (mmu4.fields.pae) |
195 | return true; | |
41cada4d | 196 | #endif /* (CONFIG_ARC_MMU_VER >= 4) */ |
75790873 EP |
197 | |
198 | return false; | |
199 | } | |
200 | ||
9f0253c6 | 201 | static inlined_cachefunc bool icache_exists(void) |
75790873 EP |
202 | { |
203 | union bcr_di_cache ibcr; | |
204 | ||
205 | ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); | |
206 | return !!ibcr.fields.ver; | |
207 | } | |
208 | ||
9f0253c6 | 209 | static inlined_cachefunc bool icache_enabled(void) |
c75eeb0b EP |
210 | { |
211 | if (!icache_exists()) | |
212 | return false; | |
213 | ||
214 | return !(read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE); | |
215 | } | |
216 | ||
9f0253c6 | 217 | static inlined_cachefunc bool dcache_exists(void) |
75790873 EP |
218 | { |
219 | union bcr_di_cache dbcr; | |
220 | ||
221 | dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); | |
222 | return !!dbcr.fields.ver; | |
223 | } | |
224 | ||
9f0253c6 | 225 | static inlined_cachefunc bool dcache_enabled(void) |
c75eeb0b EP |
226 | { |
227 | if (!dcache_exists()) | |
228 | return false; | |
229 | ||
230 | return !(read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE); | |
231 | } | |
232 | ||
9f0253c6 | 233 | static inlined_cachefunc bool slc_exists(void) |
75790873 EP |
234 | { |
235 | if (is_isa_arcv2()) { | |
236 | union bcr_generic sbcr; | |
237 | ||
238 | sbcr.word = read_aux_reg(ARC_BCR_SLC); | |
239 | return !!sbcr.fields.ver; | |
240 | } | |
241 | ||
242 | return false; | |
ef639e6f AB |
243 | } |
244 | ||
b15cb0bf EP |
245 | enum slc_dis_status { |
246 | ST_SLC_MISSING = 0, | |
247 | ST_SLC_NO_DISABLE_CTRL, | |
248 | ST_SLC_DISABLE_CTRL | |
249 | }; | |
250 | ||
251 | /* | |
252 | * ARCv1 -> ST_SLC_MISSING | |
253 | * ARCv2 && SLC absent -> ST_SLC_MISSING | |
254 | * ARCv2 && SLC exists && SLC version <= 2 -> ST_SLC_NO_DISABLE_CTRL | |
255 | * ARCv2 && SLC exists && SLC version > 2 -> ST_SLC_DISABLE_CTRL | |
256 | */ | |
257 | static inlined_cachefunc enum slc_dis_status slc_disable_supported(void) | |
258 | { | |
259 | if (is_isa_arcv2()) { | |
260 | union bcr_generic sbcr; | |
261 | ||
262 | sbcr.word = read_aux_reg(ARC_BCR_SLC); | |
263 | if (sbcr.fields.ver == 0) | |
264 | return ST_SLC_MISSING; | |
265 | else if (sbcr.fields.ver <= 2) | |
266 | return ST_SLC_NO_DISABLE_CTRL; | |
267 | else | |
268 | return ST_SLC_DISABLE_CTRL; | |
269 | } | |
270 | ||
271 | return ST_SLC_MISSING; | |
272 | } | |
273 | ||
274 | static inlined_cachefunc bool __slc_enabled(void) | |
275 | { | |
276 | return !(read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_DIS); | |
277 | } | |
278 | ||
279 | static inlined_cachefunc void __slc_enable(void) | |
280 | { | |
281 | unsigned int ctrl; | |
282 | ||
283 | ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); | |
284 | ctrl &= ~SLC_CTRL_DIS; | |
285 | write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); | |
286 | } | |
287 | ||
288 | static inlined_cachefunc void __slc_disable(void) | |
289 | { | |
290 | unsigned int ctrl; | |
291 | ||
292 | ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); | |
293 | ctrl |= SLC_CTRL_DIS; | |
294 | write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); | |
295 | } | |
296 | ||
297 | static inlined_cachefunc bool slc_enabled(void) | |
298 | { | |
299 | enum slc_dis_status slc_status = slc_disable_supported(); | |
300 | ||
301 | if (slc_status == ST_SLC_MISSING) | |
302 | return false; | |
303 | else if (slc_status == ST_SLC_NO_DISABLE_CTRL) | |
304 | return true; | |
305 | else | |
306 | return __slc_enabled(); | |
307 | } | |
308 | ||
9f0253c6 | 309 | static inlined_cachefunc bool slc_data_bypass(void) |
95336738 EP |
310 | { |
311 | /* | |
312 | * If L1 data cache is disabled SL$ is bypassed and all load/store | |
313 | * requests are sent directly to main memory. | |
314 | */ | |
315 | return !dcache_enabled(); | |
316 | } | |
317 | ||
b15cb0bf EP |
318 | void slc_enable(void) |
319 | { | |
320 | if (slc_disable_supported() != ST_SLC_DISABLE_CTRL) | |
321 | return; | |
322 | ||
323 | if (__slc_enabled()) | |
324 | return; | |
325 | ||
326 | __slc_enable(); | |
327 | } | |
328 | ||
329 | /* TODO: warn if we are not able to disable SLC */ | |
330 | void slc_disable(void) | |
331 | { | |
332 | if (slc_disable_supported() != ST_SLC_DISABLE_CTRL) | |
333 | return; | |
334 | ||
335 | /* we don't support SLC disabling if we use IOC */ | |
336 | if (ioc_enabled()) | |
337 | return; | |
338 | ||
339 | if (!__slc_enabled()) | |
340 | return; | |
341 | ||
342 | /* | |
343 | * We need to flush L1D$ to guarantee that we won't have any | |
344 | * writeback operations during SLC disabling. | |
345 | */ | |
346 | __dc_entire_op(OP_FLUSH); | |
347 | __slc_entire_op(OP_FLUSH_N_INV); | |
348 | __slc_disable(); | |
349 | } | |
350 | ||
04286d07 | 351 | static inlined_cachefunc bool ioc_exists(void) |
48b04832 EP |
352 | { |
353 | if (is_isa_arcv2()) { | |
354 | union bcr_clust_cfg cbcr; | |
355 | ||
356 | cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); | |
357 | return cbcr.fields.c; | |
358 | } | |
359 | ||
360 | return false; | |
361 | } | |
362 | ||
04286d07 | 363 | static inlined_cachefunc bool ioc_enabled(void) |
48b04832 EP |
364 | { |
365 | /* | |
366 | * We check only CONFIG option instead of IOC HW state check as IOC | |
367 | * must be disabled by default. | |
368 | */ | |
369 | if (is_ioc_enabled()) | |
370 | return ioc_exists(); | |
371 | ||
372 | return false; | |
373 | } | |
374 | ||
9f0253c6 | 375 | static inlined_cachefunc void __slc_entire_op(const int op) |
ef639e6f | 376 | { |
41cada4d EP |
377 | unsigned int ctrl; |
378 | ||
b15cb0bf | 379 | if (!slc_enabled()) |
ea9f6f1e EP |
380 | return; |
381 | ||
41cada4d | 382 | ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); |
ef639e6f | 383 | |
41cada4d EP |
384 | if (!(op & OP_FLUSH)) /* i.e. OP_INV */ |
385 | ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ | |
386 | else | |
387 | ctrl |= SLC_CTRL_IM; | |
ef639e6f | 388 | |
41cada4d | 389 | write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); |
ef639e6f | 390 | |
41cada4d EP |
391 | if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ |
392 | write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1); | |
393 | else | |
394 | write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1); | |
ef639e6f | 395 | |
41cada4d EP |
396 | /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ |
397 | read_aux_reg(ARC_AUX_SLC_CTRL); | |
ef639e6f | 398 | |
41cada4d EP |
399 | /* Important to wait for flush to complete */ |
400 | while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); | |
ef639e6f AB |
401 | } |
402 | ||
41cada4d | 403 | static void slc_upper_region_init(void) |
ef639e6f | 404 | { |
246ba284 EP |
405 | /* |
406 | * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist | |
407 | * only if PAE exists in current HW. So we had to check pae_exist | |
408 | * before using them. | |
409 | */ | |
410 | if (!pae_exists()) | |
411 | return; | |
412 | ||
41cada4d EP |
413 | /* |
414 | * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0 | |
415 | * as we don't use PAE40. | |
416 | */ | |
417 | write_aux_reg(ARC_AUX_SLC_RGN_END1, 0); | |
418 | write_aux_reg(ARC_AUX_SLC_RGN_START1, 0); | |
419 | } | |
ef639e6f | 420 | |
41cada4d EP |
421 | static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op) |
422 | { | |
05c6a26a EP |
423 | #ifdef CONFIG_ISA_ARCV2 |
424 | ||
41cada4d EP |
425 | unsigned int ctrl; |
426 | unsigned long end; | |
427 | ||
b15cb0bf | 428 | if (!slc_enabled()) |
ea9f6f1e EP |
429 | return; |
430 | ||
41cada4d EP |
431 | /* |
432 | * The Region Flush operation is specified by CTRL.RGN_OP[11..9] | |
433 | * - b'000 (default) is Flush, | |
434 | * - b'001 is Invalidate if CTRL.IM == 0 | |
435 | * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 | |
436 | */ | |
437 | ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); | |
438 | ||
439 | /* Don't rely on default value of IM bit */ | |
440 | if (!(op & OP_FLUSH)) /* i.e. OP_INV */ | |
441 | ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ | |
ef639e6f | 442 | else |
41cada4d | 443 | ctrl |= SLC_CTRL_IM; |
ef639e6f | 444 | |
41cada4d EP |
445 | if (op & OP_INV) |
446 | ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ | |
447 | else | |
448 | ctrl &= ~SLC_CTRL_RGN_OP_INV; | |
ef639e6f | 449 | |
41cada4d | 450 | write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); |
ef639e6f | 451 | |
41cada4d EP |
452 | /* |
453 | * Lower bits are ignored, no need to clip | |
454 | * END needs to be setup before START (latter triggers the operation) | |
455 | * END can't be same as START, so add (l2_line_sz - 1) to sz | |
456 | */ | |
bf8974ed | 457 | end = paddr + sz + gd->arch.slc_line_sz - 1; |
41cada4d EP |
458 | |
459 | /* | |
460 | * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1) | |
461 | * are always == 0 as we don't use PAE40, so we only setup lower ones | |
462 | * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START) | |
463 | */ | |
464 | write_aux_reg(ARC_AUX_SLC_RGN_END, end); | |
465 | write_aux_reg(ARC_AUX_SLC_RGN_START, paddr); | |
466 | ||
467 | /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ | |
468 | read_aux_reg(ARC_AUX_SLC_CTRL); | |
469 | ||
470 | while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); | |
05c6a26a EP |
471 | |
472 | #endif /* CONFIG_ISA_ARCV2 */ | |
ef639e6f | 473 | } |
a6f557c4 EP |
474 | |
475 | static void arc_ioc_setup(void) | |
476 | { | |
477 | /* IOC Aperture start is equal to DDR start */ | |
aa6e94de | 478 | unsigned int ap_base = CFG_SYS_SDRAM_BASE; |
a6f557c4 | 479 | /* IOC Aperture size is equal to DDR size */ |
aa6e94de | 480 | long ap_size = CFG_SYS_SDRAM_SIZE; |
a6f557c4 | 481 | |
7241944a EP |
482 | /* Unsupported configuration. See [ NOTE 2 ] for more details. */ |
483 | if (!slc_exists()) | |
484 | panic("Try to enable IOC but SLC is not present"); | |
485 | ||
b15cb0bf EP |
486 | if (!slc_enabled()) |
487 | panic("Try to enable IOC but SLC is disabled"); | |
488 | ||
7241944a EP |
489 | /* Unsupported configuration. See [ NOTE 2 ] for more details. */ |
490 | if (!dcache_enabled()) | |
491 | panic("Try to enable IOC but L1 D$ is disabled"); | |
492 | ||
a6f557c4 EP |
493 | if (!is_power_of_2(ap_size) || ap_size < 4096) |
494 | panic("IOC Aperture size must be power of 2 and bigger 4Kib"); | |
495 | ||
6b85b26e EP |
496 | /* IOC Aperture start must be aligned to the size of the aperture */ |
497 | if (ap_base % ap_size != 0) | |
498 | panic("IOC Aperture start must be aligned to the size of the aperture"); | |
499 | ||
500 | flush_n_invalidate_dcache_all(); | |
501 | ||
a6f557c4 EP |
502 | /* |
503 | * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, | |
504 | * so setting 0x11 implies 512M, 0x12 implies 1G... | |
505 | */ | |
506 | write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, | |
507 | order_base_2(ap_size / 1024) - 2); | |
508 | ||
a6f557c4 EP |
509 | write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12); |
510 | write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); | |
511 | write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); | |
512 | } | |
ef639e6f | 513 | |
379b3280 | 514 | static void read_decode_cache_bcr_arcv2(void) |
ef639e6f | 515 | { |
05c6a26a EP |
516 | #ifdef CONFIG_ISA_ARCV2 |
517 | ||
88ae27ed | 518 | union bcr_slc_cfg slc_cfg; |
379b3280 | 519 | |
75790873 | 520 | if (slc_exists()) { |
379b3280 | 521 | slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); |
bf8974ed | 522 | gd->arch.slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; |
7241944a EP |
523 | |
524 | /* | |
525 | * We don't support configuration where L1 I$ or L1 D$ is | |
526 | * absent but SL$ exists. See [ NOTE 2 ] for more details. | |
527 | */ | |
528 | if (!icache_exists() || !dcache_exists()) | |
529 | panic("Unsupported cache configuration: SLC exists but one of L1 caches is absent"); | |
379b3280 | 530 | } |
db6ce231 | 531 | |
05c6a26a | 532 | #endif /* CONFIG_ISA_ARCV2 */ |
ef639e6f AB |
533 | } |
534 | ||
379b3280 | 535 | void read_decode_cache_bcr(void) |
ef639e6f | 536 | { |
379b3280 | 537 | int dc_line_sz = 0, ic_line_sz = 0; |
88ae27ed | 538 | union bcr_di_cache ibcr, dbcr; |
379b3280 | 539 | |
d0a5023a AB |
540 | /* |
541 | * We don't care much about I$ line length really as there're | |
542 | * no per-line ops on I$ instead we only do full invalidation of it | |
543 | * on occasion of relocation and right before jumping to the OS. | |
544 | * Still we check insane config with zero-encoded line length in | |
545 | * presense of version field in I$ BCR. Just in case. | |
546 | */ | |
379b3280 AB |
547 | ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); |
548 | if (ibcr.fields.ver) { | |
d0a5023a | 549 | ic_line_sz = 8 << ibcr.fields.line_len; |
379b3280 AB |
550 | if (!ic_line_sz) |
551 | panic("Instruction exists but line length is 0\n"); | |
552 | } | |
553 | ||
554 | dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); | |
19b10a42 | 555 | if (dbcr.fields.ver) { |
bf8974ed | 556 | gd->arch.l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; |
379b3280 AB |
557 | if (!dc_line_sz) |
558 | panic("Data cache exists but line length is 0\n"); | |
559 | } | |
ef639e6f AB |
560 | } |
561 | ||
562 | void cache_init(void) | |
563 | { | |
379b3280 AB |
564 | read_decode_cache_bcr(); |
565 | ||
05c6a26a EP |
566 | if (is_isa_arcv2()) |
567 | read_decode_cache_bcr_arcv2(); | |
db6ce231 | 568 | |
48b04832 | 569 | if (is_isa_arcv2() && ioc_enabled()) |
a6f557c4 | 570 | arc_ioc_setup(); |
41cada4d | 571 | |
246ba284 | 572 | if (is_isa_arcv2() && slc_exists()) |
41cada4d | 573 | slc_upper_region_init(); |
ef639e6f AB |
574 | } |
575 | ||
2f16ac9d AB |
576 | int icache_status(void) |
577 | { | |
c75eeb0b | 578 | return icache_enabled(); |
2f16ac9d AB |
579 | } |
580 | ||
581 | void icache_enable(void) | |
582 | { | |
75790873 | 583 | if (icache_exists()) |
ef639e6f AB |
584 | write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & |
585 | ~IC_CTRL_CACHE_DISABLE); | |
2f16ac9d AB |
586 | } |
587 | ||
588 | void icache_disable(void) | |
589 | { | |
9f0253c6 EP |
590 | if (!icache_exists()) |
591 | return; | |
592 | ||
593 | __ic_entire_invalidate(); | |
594 | ||
595 | write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | | |
596 | IC_CTRL_CACHE_DISABLE); | |
2f16ac9d AB |
597 | } |
598 | ||
16aeee81 | 599 | /* IC supports only invalidation */ |
9f0253c6 | 600 | static inlined_cachefunc void __ic_entire_invalidate(void) |
2f16ac9d | 601 | { |
c75eeb0b | 602 | if (!icache_enabled()) |
16aeee81 EP |
603 | return; |
604 | ||
2f16ac9d | 605 | /* Any write to IC_IVIC register triggers invalidation of entire I$ */ |
16aeee81 EP |
606 | write_aux_reg(ARC_AUX_IC_IVIC, 1); |
607 | /* | |
608 | * As per ARC HS databook (see chapter 5.3.3.2) | |
609 | * it is required to add 3 NOPs after each write to IC_IVIC. | |
610 | */ | |
611 | __builtin_arc_nop(); | |
612 | __builtin_arc_nop(); | |
613 | __builtin_arc_nop(); | |
614 | read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ | |
615 | } | |
616 | ||
617 | void invalidate_icache_all(void) | |
618 | { | |
619 | __ic_entire_invalidate(); | |
41cada4d | 620 | |
95336738 EP |
621 | /* |
622 | * If SL$ is bypassed for data it is used only for instructions, | |
623 | * so we need to invalidate it too. | |
95336738 EP |
624 | */ |
625 | if (is_isa_arcv2() && slc_data_bypass()) | |
41cada4d | 626 | __slc_entire_op(OP_INV); |
41cada4d | 627 | } |
2f16ac9d AB |
628 | |
629 | int dcache_status(void) | |
630 | { | |
c75eeb0b | 631 | return dcache_enabled(); |
2f16ac9d AB |
632 | } |
633 | ||
634 | void dcache_enable(void) | |
635 | { | |
75790873 | 636 | if (!dcache_exists()) |
f8cf3d1e IG |
637 | return; |
638 | ||
2f16ac9d AB |
639 | write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & |
640 | ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); | |
641 | } | |
642 | ||
643 | void dcache_disable(void) | |
644 | { | |
75790873 | 645 | if (!dcache_exists()) |
f8cf3d1e IG |
646 | return; |
647 | ||
9f0253c6 EP |
648 | __dc_entire_op(OP_FLUSH_N_INV); |
649 | ||
650 | /* | |
651 | * As SLC will be bypassed for data after L1 D$ disable we need to | |
652 | * flush it first before L1 D$ disable. Also we invalidate SLC to | |
653 | * avoid any inconsistent data problems after enabling L1 D$ again with | |
654 | * dcache_enable function. | |
655 | */ | |
656 | if (is_isa_arcv2()) | |
657 | __slc_entire_op(OP_FLUSH_N_INV); | |
658 | ||
2f16ac9d AB |
659 | write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | |
660 | DC_CTRL_CACHE_DISABLE); | |
661 | } | |
662 | ||
c4ef14d2 EP |
663 | /* Common Helper for Line Operations on D-cache */ |
664 | static inline void __dcache_line_loop(unsigned long paddr, unsigned long sz, | |
665 | const int cacheop) | |
2f16ac9d | 666 | { |
ef639e6f | 667 | unsigned int aux_cmd; |
ef639e6f | 668 | int num_lines; |
2f16ac9d | 669 | |
c4ef14d2 EP |
670 | /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ |
671 | aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; | |
2f16ac9d | 672 | |
ef639e6f AB |
673 | sz += paddr & ~CACHE_LINE_MASK; |
674 | paddr &= CACHE_LINE_MASK; | |
2f16ac9d | 675 | |
bf8974ed | 676 | num_lines = DIV_ROUND_UP(sz, gd->arch.l1_line_sz); |
2f16ac9d | 677 | |
ef639e6f | 678 | while (num_lines-- > 0) { |
5ff40f3d | 679 | #if (CONFIG_ARC_MMU_VER == 3) |
c4ef14d2 | 680 | write_aux_reg(ARC_AUX_DC_PTAG, paddr); |
2f16ac9d | 681 | #endif |
ef639e6f | 682 | write_aux_reg(aux_cmd, paddr); |
bf8974ed | 683 | paddr += gd->arch.l1_line_sz; |
2f16ac9d | 684 | } |
2f16ac9d AB |
685 | } |
686 | ||
9f0253c6 | 687 | static inlined_cachefunc void __before_dc_op(const int op) |
2f16ac9d | 688 | { |
5d7a24d6 | 689 | unsigned int ctrl; |
ef639e6f | 690 | |
5d7a24d6 | 691 | ctrl = read_aux_reg(ARC_AUX_DC_CTRL); |
ae4a351a | 692 | |
5d7a24d6 EP |
693 | /* IM bit implies flush-n-inv, instead of vanilla inv */ |
694 | if (op == OP_INV) | |
695 | ctrl &= ~DC_CTRL_INV_MODE_FLUSH; | |
696 | else | |
697 | ctrl |= DC_CTRL_INV_MODE_FLUSH; | |
698 | ||
699 | write_aux_reg(ARC_AUX_DC_CTRL, ctrl); | |
2f16ac9d AB |
700 | } |
701 | ||
9f0253c6 | 702 | static inlined_cachefunc void __after_dc_op(const int op) |
2f16ac9d | 703 | { |
ef639e6f | 704 | if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ |
19b10a42 | 705 | while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS); |
2f16ac9d | 706 | } |
6eb15e50 | 707 | |
9f0253c6 | 708 | static inlined_cachefunc void __dc_entire_op(const int cacheop) |
6eb15e50 | 709 | { |
ef639e6f | 710 | int aux; |
5d7a24d6 | 711 | |
c75eeb0b | 712 | if (!dcache_enabled()) |
c877a891 EP |
713 | return; |
714 | ||
5d7a24d6 | 715 | __before_dc_op(cacheop); |
6eb15e50 | 716 | |
ef639e6f AB |
717 | if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ |
718 | aux = ARC_AUX_DC_IVDC; | |
719 | else | |
720 | aux = ARC_AUX_DC_FLSH; | |
6eb15e50 | 721 | |
ef639e6f | 722 | write_aux_reg(aux, 0x1); |
6eb15e50 | 723 | |
5d7a24d6 | 724 | __after_dc_op(cacheop); |
6eb15e50 AB |
725 | } |
726 | ||
ef639e6f AB |
727 | static inline void __dc_line_op(unsigned long paddr, unsigned long sz, |
728 | const int cacheop) | |
6eb15e50 | 729 | { |
c75eeb0b | 730 | if (!dcache_enabled()) |
c877a891 EP |
731 | return; |
732 | ||
5d7a24d6 | 733 | __before_dc_op(cacheop); |
c4ef14d2 | 734 | __dcache_line_loop(paddr, sz, cacheop); |
5d7a24d6 | 735 | __after_dc_op(cacheop); |
ef639e6f | 736 | } |
6eb15e50 | 737 | |
ef639e6f AB |
738 | void invalidate_dcache_range(unsigned long start, unsigned long end) |
739 | { | |
41cada4d EP |
740 | if (start >= end) |
741 | return; | |
742 | ||
05c6a26a | 743 | /* |
95336738 EP |
744 | * ARCv1 -> call __dc_line_op |
745 | * ARCv2 && L1 D$ disabled -> nothing | |
746 | * ARCv2 && L1 D$ enabled && IOC enabled -> nothing | |
747 | * ARCv2 && L1 D$ enabled && no IOC -> call __dc_line_op; call __slc_rgn_op | |
05c6a26a | 748 | */ |
48b04832 | 749 | if (!is_isa_arcv2() || !ioc_enabled()) |
db6ce231 AB |
750 | __dc_line_op(start, end - start, OP_INV); |
751 | ||
95336738 | 752 | if (is_isa_arcv2() && !ioc_enabled() && !slc_data_bypass()) |
41cada4d | 753 | __slc_rgn_op(start, end - start, OP_INV); |
ef639e6f | 754 | } |
6eb15e50 | 755 | |
ef639e6f AB |
756 | void flush_dcache_range(unsigned long start, unsigned long end) |
757 | { | |
41cada4d EP |
758 | if (start >= end) |
759 | return; | |
760 | ||
05c6a26a | 761 | /* |
95336738 EP |
762 | * ARCv1 -> call __dc_line_op |
763 | * ARCv2 && L1 D$ disabled -> nothing | |
764 | * ARCv2 && L1 D$ enabled && IOC enabled -> nothing | |
765 | * ARCv2 && L1 D$ enabled && no IOC -> call __dc_line_op; call __slc_rgn_op | |
05c6a26a | 766 | */ |
48b04832 | 767 | if (!is_isa_arcv2() || !ioc_enabled()) |
db6ce231 AB |
768 | __dc_line_op(start, end - start, OP_FLUSH); |
769 | ||
95336738 | 770 | if (is_isa_arcv2() && !ioc_enabled() && !slc_data_bypass()) |
41cada4d | 771 | __slc_rgn_op(start, end - start, OP_FLUSH); |
6eb15e50 AB |
772 | } |
773 | ||
ef639e6f | 774 | void flush_cache(unsigned long start, unsigned long size) |
6eb15e50 | 775 | { |
ef639e6f AB |
776 | flush_dcache_range(start, start + size); |
777 | } | |
6eb15e50 | 778 | |
c27814be EP |
779 | /* |
780 | * As invalidate_dcache_all() is not used in generic U-Boot code and as we | |
781 | * don't need it in arch/arc code alone (invalidate without flush) we implement | |
782 | * flush_n_invalidate_dcache_all (flush and invalidate in 1 operation) because | |
783 | * it's much safer. See [ NOTE 1 ] for more details. | |
784 | */ | |
785 | void flush_n_invalidate_dcache_all(void) | |
ef639e6f | 786 | { |
c27814be | 787 | __dc_entire_op(OP_FLUSH_N_INV); |
db6ce231 | 788 | |
95336738 | 789 | if (is_isa_arcv2() && !slc_data_bypass()) |
c27814be | 790 | __slc_entire_op(OP_FLUSH_N_INV); |
6eb15e50 AB |
791 | } |
792 | ||
ef639e6f AB |
793 | void flush_dcache_all(void) |
794 | { | |
2a8382c6 | 795 | __dc_entire_op(OP_FLUSH); |
db6ce231 | 796 | |
95336738 | 797 | if (is_isa_arcv2() && !slc_data_bypass()) |
ef639e6f | 798 | __slc_entire_op(OP_FLUSH); |
ef639e6f | 799 | } |
375945ba EP |
800 | |
801 | /* | |
802 | * This is function to cleanup all caches (and therefore sync I/D caches) which | |
803 | * can be used for cleanup before linux launch or to sync caches during | |
804 | * relocation. | |
805 | */ | |
806 | void sync_n_cleanup_cache_all(void) | |
807 | { | |
808 | __dc_entire_op(OP_FLUSH_N_INV); | |
809 | ||
810 | /* | |
811 | * If SL$ is bypassed for data it is used only for instructions, | |
812 | * and we shouldn't flush it. So invalidate it instead of flush_n_inv. | |
813 | */ | |
814 | if (is_isa_arcv2()) { | |
815 | if (slc_data_bypass()) | |
816 | __slc_entire_op(OP_INV); | |
817 | else | |
818 | __slc_entire_op(OP_FLUSH_N_INV); | |
819 | } | |
820 | ||
821 | __ic_entire_invalidate(); | |
822 | } | |
cfa19719 MV |
823 | |
824 | static ulong get_sp(void) | |
825 | { | |
826 | ulong ret; | |
827 | ||
828 | asm("mov %0, sp" : "=r"(ret) : ); | |
829 | return ret; | |
830 | } | |
831 | ||
832 | void arch_lmb_reserve(struct lmb *lmb) | |
833 | { | |
1f391c34 | 834 | arch_lmb_reserve_generic(lmb, get_sp(), gd->ram_top, 4096); |
cfa19719 | 835 | } |