]>
Commit | Line | Data |
---|---|---|
2c451f78 A |
1 | /* |
2 | * (C) Copyright 2010 | |
3 | * Texas Instruments, <www.ti.com> | |
4 | * Aneesh V <aneesh@ti.com> | |
5 | * | |
6 | * See file CREDITS for list of people who contributed to this | |
7 | * project. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License as | |
11 | * published by the Free Software Foundation; either version 2 of | |
12 | * the License, or (at your option) any later version. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | * GNU General Public License for more details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License | |
20 | * along with this program; if not, write to the Free Software | |
21 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, | |
22 | * MA 02111-1307 USA | |
23 | */ | |
24 | #include <linux/types.h> | |
25 | #include <common.h> | |
26 | #include <asm/armv7.h> | |
27 | #include <asm/utils.h> | |
28 | ||
29 | #define ARMV7_DCACHE_INVAL_ALL 1 | |
30 | #define ARMV7_DCACHE_CLEAN_INVAL_ALL 2 | |
31 | #define ARMV7_DCACHE_INVAL_RANGE 3 | |
32 | #define ARMV7_DCACHE_CLEAN_INVAL_RANGE 4 | |
33 | ||
34 | #ifndef CONFIG_SYS_DCACHE_OFF | |
35 | /* | |
36 | * Write the level and type you want to Cache Size Selection Register(CSSELR) | |
37 | * to get size details from Current Cache Size ID Register(CCSIDR) | |
38 | */ | |
39 | static void set_csselr(u32 level, u32 type) | |
40 | { u32 csselr = level << 1 | type; | |
41 | ||
42 | /* Write to Cache Size Selection Register(CSSELR) */ | |
43 | asm volatile ("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr)); | |
44 | } | |
45 | ||
46 | static u32 get_ccsidr(void) | |
47 | { | |
48 | u32 ccsidr; | |
49 | ||
50 | /* Read current CP15 Cache Size ID Register */ | |
51 | asm volatile ("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr)); | |
52 | return ccsidr; | |
53 | } | |
54 | ||
55 | static u32 get_clidr(void) | |
56 | { | |
57 | u32 clidr; | |
58 | ||
59 | /* Read current CP15 Cache Level ID Register */ | |
60 | asm volatile ("mrc p15,1,%0,c0,c0,1" : "=r" (clidr)); | |
61 | return clidr; | |
62 | } | |
63 | ||
64 | static void v7_inval_dcache_level_setway(u32 level, u32 num_sets, | |
65 | u32 num_ways, u32 way_shift, | |
66 | u32 log2_line_len) | |
67 | { | |
68 | int way, set, setway; | |
69 | ||
70 | /* | |
71 | * For optimal assembly code: | |
72 | * a. count down | |
73 | * b. have bigger loop inside | |
74 | */ | |
75 | for (way = num_ways - 1; way >= 0 ; way--) { | |
76 | for (set = num_sets - 1; set >= 0; set--) { | |
77 | setway = (level << 1) | (set << log2_line_len) | | |
78 | (way << way_shift); | |
79 | /* Invalidate data/unified cache line by set/way */ | |
80 | asm volatile (" mcr p15, 0, %0, c7, c6, 2" | |
81 | : : "r" (setway)); | |
82 | } | |
83 | } | |
882f80b9 A |
84 | /* DSB to make sure the operation is complete */ |
85 | CP15DSB; | |
2c451f78 A |
86 | } |
87 | ||
88 | static void v7_clean_inval_dcache_level_setway(u32 level, u32 num_sets, | |
89 | u32 num_ways, u32 way_shift, | |
90 | u32 log2_line_len) | |
91 | { | |
92 | int way, set, setway; | |
93 | ||
94 | /* | |
95 | * For optimal assembly code: | |
96 | * a. count down | |
97 | * b. have bigger loop inside | |
98 | */ | |
99 | for (way = num_ways - 1; way >= 0 ; way--) { | |
100 | for (set = num_sets - 1; set >= 0; set--) { | |
101 | setway = (level << 1) | (set << log2_line_len) | | |
102 | (way << way_shift); | |
103 | /* | |
104 | * Clean & Invalidate data/unified | |
105 | * cache line by set/way | |
106 | */ | |
107 | asm volatile (" mcr p15, 0, %0, c7, c14, 2" | |
108 | : : "r" (setway)); | |
109 | } | |
110 | } | |
882f80b9 A |
111 | /* DSB to make sure the operation is complete */ |
112 | CP15DSB; | |
2c451f78 A |
113 | } |
114 | ||
115 | static void v7_maint_dcache_level_setway(u32 level, u32 operation) | |
116 | { | |
117 | u32 ccsidr; | |
118 | u32 num_sets, num_ways, log2_line_len, log2_num_ways; | |
119 | u32 way_shift; | |
120 | ||
121 | set_csselr(level, ARMV7_CSSELR_IND_DATA_UNIFIED); | |
122 | ||
123 | ccsidr = get_ccsidr(); | |
124 | ||
125 | log2_line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >> | |
126 | CCSIDR_LINE_SIZE_OFFSET) + 2; | |
127 | /* Converting from words to bytes */ | |
128 | log2_line_len += 2; | |
129 | ||
130 | num_ways = ((ccsidr & CCSIDR_ASSOCIATIVITY_MASK) >> | |
131 | CCSIDR_ASSOCIATIVITY_OFFSET) + 1; | |
132 | num_sets = ((ccsidr & CCSIDR_NUM_SETS_MASK) >> | |
133 | CCSIDR_NUM_SETS_OFFSET) + 1; | |
134 | /* | |
135 | * According to ARMv7 ARM number of sets and number of ways need | |
136 | * not be a power of 2 | |
137 | */ | |
138 | log2_num_ways = log_2_n_round_up(num_ways); | |
139 | ||
140 | way_shift = (32 - log2_num_ways); | |
141 | if (operation == ARMV7_DCACHE_INVAL_ALL) { | |
142 | v7_inval_dcache_level_setway(level, num_sets, num_ways, | |
143 | way_shift, log2_line_len); | |
144 | } else if (operation == ARMV7_DCACHE_CLEAN_INVAL_ALL) { | |
145 | v7_clean_inval_dcache_level_setway(level, num_sets, num_ways, | |
146 | way_shift, log2_line_len); | |
147 | } | |
148 | } | |
149 | ||
150 | static void v7_maint_dcache_all(u32 operation) | |
151 | { | |
152 | u32 level, cache_type, level_start_bit = 0; | |
153 | ||
154 | u32 clidr = get_clidr(); | |
155 | ||
156 | for (level = 0; level < 7; level++) { | |
157 | cache_type = (clidr >> level_start_bit) & 0x7; | |
158 | if ((cache_type == ARMV7_CLIDR_CTYPE_DATA_ONLY) || | |
159 | (cache_type == ARMV7_CLIDR_CTYPE_INSTRUCTION_DATA) || | |
160 | (cache_type == ARMV7_CLIDR_CTYPE_UNIFIED)) | |
161 | v7_maint_dcache_level_setway(level, operation); | |
162 | level_start_bit += 3; | |
163 | } | |
164 | } | |
165 | ||
166 | static void v7_dcache_clean_inval_range(u32 start, | |
167 | u32 stop, u32 line_len) | |
168 | { | |
169 | u32 mva; | |
170 | ||
171 | /* Align start to cache line boundary */ | |
172 | start &= ~(line_len - 1); | |
173 | for (mva = start; mva < stop; mva = mva + line_len) { | |
174 | /* DCCIMVAC - Clean & Invalidate data cache by MVA to PoC */ | |
175 | asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva)); | |
176 | } | |
177 | } | |
178 | ||
179 | static void v7_dcache_inval_range(u32 start, u32 stop, u32 line_len) | |
180 | { | |
181 | u32 mva; | |
182 | ||
183 | /* | |
cabe2878 A |
184 | * If start address is not aligned to cache-line do not |
185 | * invalidate the first cache-line | |
2c451f78 A |
186 | */ |
187 | if (start & (line_len - 1)) { | |
cabe2878 A |
188 | printf("ERROR: %s - start address is not aligned - 0x%08x\n", |
189 | __func__, start); | |
2c451f78 A |
190 | /* move to next cache line */ |
191 | start = (start + line_len - 1) & ~(line_len - 1); | |
192 | } | |
193 | ||
194 | /* | |
cabe2878 A |
195 | * If stop address is not aligned to cache-line do not |
196 | * invalidate the last cache-line | |
2c451f78 A |
197 | */ |
198 | if (stop & (line_len - 1)) { | |
cabe2878 A |
199 | printf("ERROR: %s - stop address is not aligned - 0x%08x\n", |
200 | __func__, stop); | |
2c451f78 A |
201 | /* align to the beginning of this cache line */ |
202 | stop &= ~(line_len - 1); | |
203 | } | |
204 | ||
205 | for (mva = start; mva < stop; mva = mva + line_len) { | |
206 | /* DCIMVAC - Invalidate data cache by MVA to PoC */ | |
207 | asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva)); | |
208 | } | |
209 | } | |
210 | ||
211 | static void v7_dcache_maint_range(u32 start, u32 stop, u32 range_op) | |
212 | { | |
213 | u32 line_len, ccsidr; | |
214 | ||
215 | ccsidr = get_ccsidr(); | |
216 | line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >> | |
217 | CCSIDR_LINE_SIZE_OFFSET) + 2; | |
218 | /* Converting from words to bytes */ | |
219 | line_len += 2; | |
220 | /* converting from log2(linelen) to linelen */ | |
221 | line_len = 1 << line_len; | |
222 | ||
223 | switch (range_op) { | |
224 | case ARMV7_DCACHE_CLEAN_INVAL_RANGE: | |
225 | v7_dcache_clean_inval_range(start, stop, line_len); | |
226 | break; | |
227 | case ARMV7_DCACHE_INVAL_RANGE: | |
228 | v7_dcache_inval_range(start, stop, line_len); | |
229 | break; | |
230 | } | |
231 | ||
882f80b9 A |
232 | /* DSB to make sure the operation is complete */ |
233 | CP15DSB; | |
2c451f78 A |
234 | } |
235 | ||
236 | /* Invalidate TLB */ | |
237 | static void v7_inval_tlb(void) | |
238 | { | |
239 | /* Invalidate entire unified TLB */ | |
240 | asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0)); | |
241 | /* Invalidate entire data TLB */ | |
242 | asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0)); | |
243 | /* Invalidate entire instruction TLB */ | |
244 | asm volatile ("mcr p15, 0, %0, c8, c5, 0" : : "r" (0)); | |
245 | /* Full system DSB - make sure that the invalidation is complete */ | |
246 | CP15DSB; | |
247 | /* Full system ISB - make sure the instruction stream sees it */ | |
248 | CP15ISB; | |
249 | } | |
250 | ||
251 | void invalidate_dcache_all(void) | |
252 | { | |
253 | v7_maint_dcache_all(ARMV7_DCACHE_INVAL_ALL); | |
254 | ||
255 | v7_outer_cache_inval_all(); | |
256 | } | |
257 | ||
258 | /* | |
259 | * Performs a clean & invalidation of the entire data cache | |
260 | * at all levels | |
261 | */ | |
262 | void flush_dcache_all(void) | |
263 | { | |
264 | v7_maint_dcache_all(ARMV7_DCACHE_CLEAN_INVAL_ALL); | |
265 | ||
266 | v7_outer_cache_flush_all(); | |
267 | } | |
268 | ||
269 | /* | |
270 | * Invalidates range in all levels of D-cache/unified cache used: | |
271 | * Affects the range [start, stop - 1] | |
272 | */ | |
273 | void invalidate_dcache_range(unsigned long start, unsigned long stop) | |
274 | { | |
275 | ||
276 | v7_dcache_maint_range(start, stop, ARMV7_DCACHE_INVAL_RANGE); | |
277 | ||
278 | v7_outer_cache_inval_range(start, stop); | |
279 | } | |
280 | ||
281 | /* | |
282 | * Flush range(clean & invalidate) from all levels of D-cache/unified | |
283 | * cache used: | |
284 | * Affects the range [start, stop - 1] | |
285 | */ | |
286 | void flush_dcache_range(unsigned long start, unsigned long stop) | |
287 | { | |
288 | v7_dcache_maint_range(start, stop, ARMV7_DCACHE_CLEAN_INVAL_RANGE); | |
289 | ||
290 | v7_outer_cache_flush_range(start, stop); | |
291 | } | |
292 | ||
293 | void arm_init_before_mmu(void) | |
294 | { | |
295 | v7_outer_cache_enable(); | |
296 | invalidate_dcache_all(); | |
297 | v7_inval_tlb(); | |
298 | } | |
299 | ||
300 | /* | |
301 | * Flush range from all levels of d-cache/unified-cache used: | |
302 | * Affects the range [start, start + size - 1] | |
303 | */ | |
304 | void flush_cache(unsigned long start, unsigned long size) | |
305 | { | |
306 | flush_dcache_range(start, start + size); | |
307 | } | |
308 | #else /* #ifndef CONFIG_SYS_DCACHE_OFF */ | |
309 | void invalidate_dcache_all(void) | |
310 | { | |
311 | } | |
312 | ||
313 | void flush_dcache_all(void) | |
314 | { | |
315 | } | |
316 | ||
317 | void invalidate_dcache_range(unsigned long start, unsigned long stop) | |
318 | { | |
319 | } | |
320 | ||
321 | void flush_dcache_range(unsigned long start, unsigned long stop) | |
322 | { | |
323 | } | |
324 | ||
325 | void arm_init_before_mmu(void) | |
326 | { | |
327 | } | |
328 | ||
329 | void flush_cache(unsigned long start, unsigned long size) | |
330 | { | |
331 | } | |
332 | #endif /* #ifndef CONFIG_SYS_DCACHE_OFF */ | |
333 | ||
334 | #ifndef CONFIG_SYS_ICACHE_OFF | |
335 | /* Invalidate entire I-cache and branch predictor array */ | |
336 | void invalidate_icache_all(void) | |
337 | { | |
338 | /* | |
339 | * Invalidate all instruction caches to PoU. | |
340 | * Also flushes branch target cache. | |
341 | */ | |
342 | asm volatile ("mcr p15, 0, %0, c7, c5, 0" : : "r" (0)); | |
343 | ||
344 | /* Invalidate entire branch predictor array */ | |
345 | asm volatile ("mcr p15, 0, %0, c7, c5, 6" : : "r" (0)); | |
346 | ||
347 | /* Full system DSB - make sure that the invalidation is complete */ | |
348 | CP15DSB; | |
349 | ||
350 | /* ISB - make sure the instruction stream sees it */ | |
351 | CP15ISB; | |
352 | } | |
353 | #else | |
354 | void invalidate_icache_all(void) | |
355 | { | |
356 | } | |
357 | #endif | |
358 | ||
359 | /* | |
360 | * Stub implementations for outer cache operations | |
361 | */ | |
362 | void __v7_outer_cache_enable(void) | |
363 | { | |
364 | } | |
365 | void v7_outer_cache_enable(void) | |
366 | __attribute__((weak, alias("__v7_outer_cache_enable"))); | |
367 | ||
368 | void __v7_outer_cache_disable(void) | |
369 | { | |
370 | } | |
371 | void v7_outer_cache_disable(void) | |
372 | __attribute__((weak, alias("__v7_outer_cache_disable"))); | |
373 | ||
374 | void __v7_outer_cache_flush_all(void) | |
375 | { | |
376 | } | |
377 | void v7_outer_cache_flush_all(void) | |
378 | __attribute__((weak, alias("__v7_outer_cache_flush_all"))); | |
379 | ||
380 | void __v7_outer_cache_inval_all(void) | |
381 | { | |
382 | } | |
383 | void v7_outer_cache_inval_all(void) | |
384 | __attribute__((weak, alias("__v7_outer_cache_inval_all"))); | |
385 | ||
386 | void __v7_outer_cache_flush_range(u32 start, u32 end) | |
387 | { | |
388 | } | |
389 | void v7_outer_cache_flush_range(u32 start, u32 end) | |
390 | __attribute__((weak, alias("__v7_outer_cache_flush_range"))); | |
391 | ||
392 | void __v7_outer_cache_inval_range(u32 start, u32 end) | |
393 | { | |
394 | } | |
395 | void v7_outer_cache_inval_range(u32 start, u32 end) | |
396 | __attribute__((weak, alias("__v7_outer_cache_inval_range"))); |