]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
88278ca2 | 2 | /* |
1da177e4 LT |
3 | * hypersparc.S: High speed Hypersparc mmu/cache operations. |
4 | * | |
5 | * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) | |
6 | */ | |
7 | ||
8 | #include <asm/ptrace.h> | |
9 | #include <asm/psr.h> | |
47003497 | 10 | #include <asm/asm-offsets.h> |
1da177e4 LT |
11 | #include <asm/asi.h> |
12 | #include <asm/page.h> | |
13 | #include <asm/pgtsrmmu.h> | |
1da177e4 LT |
14 | #include <linux/init.h> |
15 | ||
16 | .text | |
17 | .align 4 | |
18 | ||
19 | .globl hypersparc_flush_cache_all, hypersparc_flush_cache_mm | |
20 | .globl hypersparc_flush_cache_range, hypersparc_flush_cache_page | |
21 | .globl hypersparc_flush_page_to_ram | |
22 | .globl hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns | |
23 | .globl hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm | |
24 | .globl hypersparc_flush_tlb_range, hypersparc_flush_tlb_page | |
25 | ||
26 | hypersparc_flush_cache_all: | |
27 | WINDOW_FLUSH(%g4, %g5) | |
28 | sethi %hi(vac_cache_size), %g4 | |
29 | ld [%g4 + %lo(vac_cache_size)], %g5 | |
30 | sethi %hi(vac_line_size), %g1 | |
31 | ld [%g1 + %lo(vac_line_size)], %g2 | |
32 | 1: | |
33 | subcc %g5, %g2, %g5 ! hyper_flush_unconditional_combined | |
34 | bne 1b | |
35 | sta %g0, [%g5] ASI_M_FLUSH_CTX | |
36 | retl | |
37 | sta %g0, [%g0] ASI_M_FLUSH_IWHOLE ! hyper_flush_whole_icache | |
38 | ||
39 | /* We expand the window flush to get maximum performance. */ | |
40 | hypersparc_flush_cache_mm: | |
41 | #ifndef CONFIG_SMP | |
42 | ld [%o0 + AOFF_mm_context], %g1 | |
43 | cmp %g1, -1 | |
44 | be hypersparc_flush_cache_mm_out | |
45 | #endif | |
46 | WINDOW_FLUSH(%g4, %g5) | |
47 | ||
48 | sethi %hi(vac_line_size), %g1 | |
49 | ld [%g1 + %lo(vac_line_size)], %o1 | |
50 | sethi %hi(vac_cache_size), %g2 | |
51 | ld [%g2 + %lo(vac_cache_size)], %o0 | |
52 | add %o1, %o1, %g1 | |
53 | add %o1, %g1, %g2 | |
54 | add %o1, %g2, %g3 | |
55 | add %o1, %g3, %g4 | |
56 | add %o1, %g4, %g5 | |
57 | add %o1, %g5, %o4 | |
58 | add %o1, %o4, %o5 | |
59 | ||
60 | /* BLAMMO! */ | |
61 | 1: | |
62 | subcc %o0, %o5, %o0 ! hyper_flush_cache_user | |
63 | sta %g0, [%o0 + %g0] ASI_M_FLUSH_USER | |
64 | sta %g0, [%o0 + %o1] ASI_M_FLUSH_USER | |
65 | sta %g0, [%o0 + %g1] ASI_M_FLUSH_USER | |
66 | sta %g0, [%o0 + %g2] ASI_M_FLUSH_USER | |
67 | sta %g0, [%o0 + %g3] ASI_M_FLUSH_USER | |
68 | sta %g0, [%o0 + %g4] ASI_M_FLUSH_USER | |
69 | sta %g0, [%o0 + %g5] ASI_M_FLUSH_USER | |
70 | bne 1b | |
71 | sta %g0, [%o0 + %o4] ASI_M_FLUSH_USER | |
72 | hypersparc_flush_cache_mm_out: | |
73 | retl | |
74 | nop | |
75 | ||
76 | /* The things we do for performance... */ | |
77 | hypersparc_flush_cache_range: | |
961246b4 | 78 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
79 | #ifndef CONFIG_SMP |
80 | ld [%o0 + AOFF_mm_context], %g1 | |
81 | cmp %g1, -1 | |
82 | be hypersparc_flush_cache_range_out | |
83 | #endif | |
84 | WINDOW_FLUSH(%g4, %g5) | |
85 | ||
86 | sethi %hi(vac_line_size), %g1 | |
87 | ld [%g1 + %lo(vac_line_size)], %o4 | |
88 | sethi %hi(vac_cache_size), %g2 | |
89 | ld [%g2 + %lo(vac_cache_size)], %o3 | |
90 | ||
91 | /* Here comes the fun part... */ | |
92 | add %o2, (PAGE_SIZE - 1), %o2 | |
93 | andn %o1, (PAGE_SIZE - 1), %o1 | |
94 | add %o4, %o4, %o5 | |
95 | andn %o2, (PAGE_SIZE - 1), %o2 | |
96 | add %o4, %o5, %g1 | |
97 | sub %o2, %o1, %g4 | |
98 | add %o4, %g1, %g2 | |
99 | sll %o3, 2, %g5 | |
100 | add %o4, %g2, %g3 | |
101 | cmp %g4, %g5 | |
102 | add %o4, %g3, %g4 | |
103 | blu 0f | |
104 | add %o4, %g4, %g5 | |
105 | add %o4, %g5, %g7 | |
106 | ||
107 | /* Flush entire user space, believe it or not this is quicker | |
108 | * than page at a time flushings for range > (cache_size<<2). | |
109 | */ | |
110 | 1: | |
111 | subcc %o3, %g7, %o3 | |
112 | sta %g0, [%o3 + %g0] ASI_M_FLUSH_USER | |
113 | sta %g0, [%o3 + %o4] ASI_M_FLUSH_USER | |
114 | sta %g0, [%o3 + %o5] ASI_M_FLUSH_USER | |
115 | sta %g0, [%o3 + %g1] ASI_M_FLUSH_USER | |
116 | sta %g0, [%o3 + %g2] ASI_M_FLUSH_USER | |
117 | sta %g0, [%o3 + %g3] ASI_M_FLUSH_USER | |
118 | sta %g0, [%o3 + %g4] ASI_M_FLUSH_USER | |
119 | bne 1b | |
120 | sta %g0, [%o3 + %g5] ASI_M_FLUSH_USER | |
121 | retl | |
122 | nop | |
123 | ||
124 | /* Below our threshold, flush one page at a time. */ | |
125 | 0: | |
126 | ld [%o0 + AOFF_mm_context], %o0 | |
127 | mov SRMMU_CTX_REG, %g7 | |
128 | lda [%g7] ASI_M_MMUREGS, %o3 | |
129 | sta %o0, [%g7] ASI_M_MMUREGS | |
130 | add %o2, -PAGE_SIZE, %o0 | |
131 | 1: | |
132 | or %o0, 0x400, %g7 | |
133 | lda [%g7] ASI_M_FLUSH_PROBE, %g7 | |
134 | orcc %g7, 0, %g0 | |
135 | be,a 3f | |
136 | mov %o0, %o2 | |
137 | add %o4, %g5, %g7 | |
138 | 2: | |
139 | sub %o2, %g7, %o2 | |
140 | sta %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE | |
141 | sta %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE | |
142 | sta %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE | |
143 | sta %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE | |
144 | sta %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE | |
145 | sta %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE | |
146 | andcc %o2, 0xffc, %g0 | |
147 | sta %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE | |
148 | bne 2b | |
149 | sta %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE | |
150 | 3: | |
151 | cmp %o2, %o1 | |
152 | bne 1b | |
153 | add %o2, -PAGE_SIZE, %o0 | |
154 | mov SRMMU_FAULT_STATUS, %g5 | |
155 | lda [%g5] ASI_M_MMUREGS, %g0 | |
156 | mov SRMMU_CTX_REG, %g7 | |
157 | sta %o3, [%g7] ASI_M_MMUREGS | |
158 | hypersparc_flush_cache_range_out: | |
159 | retl | |
160 | nop | |
161 | ||
162 | /* HyperSparc requires a valid mapping where we are about to flush | |
163 | * in order to check for a physical tag match during the flush. | |
164 | */ | |
165 | /* Verified, my ass... */ | |
166 | hypersparc_flush_cache_page: | |
961246b4 | 167 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
168 | ld [%o0 + AOFF_mm_context], %g2 |
169 | #ifndef CONFIG_SMP | |
170 | cmp %g2, -1 | |
171 | be hypersparc_flush_cache_page_out | |
172 | #endif | |
173 | WINDOW_FLUSH(%g4, %g5) | |
174 | ||
175 | sethi %hi(vac_line_size), %g1 | |
176 | ld [%g1 + %lo(vac_line_size)], %o4 | |
177 | mov SRMMU_CTX_REG, %o3 | |
178 | andn %o1, (PAGE_SIZE - 1), %o1 | |
179 | lda [%o3] ASI_M_MMUREGS, %o2 | |
180 | sta %g2, [%o3] ASI_M_MMUREGS | |
181 | or %o1, 0x400, %o5 | |
182 | lda [%o5] ASI_M_FLUSH_PROBE, %g1 | |
183 | orcc %g0, %g1, %g0 | |
184 | be 2f | |
185 | add %o4, %o4, %o5 | |
186 | sub %o1, -PAGE_SIZE, %o1 | |
187 | add %o4, %o5, %g1 | |
188 | add %o4, %g1, %g2 | |
189 | add %o4, %g2, %g3 | |
190 | add %o4, %g3, %g4 | |
191 | add %o4, %g4, %g5 | |
192 | add %o4, %g5, %g7 | |
193 | ||
194 | /* BLAMMO! */ | |
195 | 1: | |
196 | sub %o1, %g7, %o1 | |
197 | sta %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE | |
198 | sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE | |
199 | sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE | |
200 | sta %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE | |
201 | sta %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE | |
202 | sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE | |
203 | andcc %o1, 0xffc, %g0 | |
204 | sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE | |
205 | bne 1b | |
206 | sta %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE | |
207 | 2: | |
208 | mov SRMMU_FAULT_STATUS, %g7 | |
209 | mov SRMMU_CTX_REG, %g4 | |
210 | lda [%g7] ASI_M_MMUREGS, %g0 | |
211 | sta %o2, [%g4] ASI_M_MMUREGS | |
212 | hypersparc_flush_cache_page_out: | |
213 | retl | |
214 | nop | |
215 | ||
216 | hypersparc_flush_sig_insns: | |
217 | flush %o1 | |
218 | retl | |
219 | flush %o1 + 4 | |
220 | ||
221 | /* HyperSparc is copy-back. */ | |
222 | hypersparc_flush_page_to_ram: | |
223 | sethi %hi(vac_line_size), %g1 | |
224 | ld [%g1 + %lo(vac_line_size)], %o4 | |
225 | andn %o0, (PAGE_SIZE - 1), %o0 | |
226 | add %o4, %o4, %o5 | |
227 | or %o0, 0x400, %g7 | |
228 | lda [%g7] ASI_M_FLUSH_PROBE, %g5 | |
229 | add %o4, %o5, %g1 | |
230 | orcc %g5, 0, %g0 | |
231 | be 2f | |
232 | add %o4, %g1, %g2 | |
233 | add %o4, %g2, %g3 | |
234 | sub %o0, -PAGE_SIZE, %o0 | |
235 | add %o4, %g3, %g4 | |
236 | add %o4, %g4, %g5 | |
237 | add %o4, %g5, %g7 | |
238 | ||
239 | /* BLAMMO! */ | |
240 | 1: | |
241 | sub %o0, %g7, %o0 | |
242 | sta %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE | |
243 | sta %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE | |
244 | sta %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE | |
245 | sta %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE | |
246 | sta %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE | |
247 | sta %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE | |
248 | andcc %o0, 0xffc, %g0 | |
249 | sta %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE | |
250 | bne 1b | |
251 | sta %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE | |
252 | 2: | |
253 | mov SRMMU_FAULT_STATUS, %g1 | |
254 | retl | |
255 | lda [%g1] ASI_M_MMUREGS, %g0 | |
256 | ||
257 | /* HyperSparc is IO cache coherent. */ | |
258 | hypersparc_flush_page_for_dma: | |
259 | retl | |
260 | nop | |
261 | ||
262 | /* It was noted that at boot time a TLB flush all in a delay slot | |
263 | * can deliver an illegal instruction to the processor if the timing | |
264 | * is just right... | |
265 | */ | |
266 | hypersparc_flush_tlb_all: | |
267 | mov 0x400, %g1 | |
268 | sta %g0, [%g1] ASI_M_FLUSH_PROBE | |
269 | retl | |
270 | nop | |
271 | ||
272 | hypersparc_flush_tlb_mm: | |
273 | mov SRMMU_CTX_REG, %g1 | |
274 | ld [%o0 + AOFF_mm_context], %o1 | |
275 | lda [%g1] ASI_M_MMUREGS, %g5 | |
276 | #ifndef CONFIG_SMP | |
277 | cmp %o1, -1 | |
278 | be hypersparc_flush_tlb_mm_out | |
279 | #endif | |
280 | mov 0x300, %g2 | |
281 | sta %o1, [%g1] ASI_M_MMUREGS | |
282 | sta %g0, [%g2] ASI_M_FLUSH_PROBE | |
283 | hypersparc_flush_tlb_mm_out: | |
284 | retl | |
285 | sta %g5, [%g1] ASI_M_MMUREGS | |
286 | ||
287 | hypersparc_flush_tlb_range: | |
961246b4 | 288 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
289 | mov SRMMU_CTX_REG, %g1 |
290 | ld [%o0 + AOFF_mm_context], %o3 | |
291 | lda [%g1] ASI_M_MMUREGS, %g5 | |
292 | #ifndef CONFIG_SMP | |
293 | cmp %o3, -1 | |
294 | be hypersparc_flush_tlb_range_out | |
295 | #endif | |
296 | sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 | |
297 | sta %o3, [%g1] ASI_M_MMUREGS | |
298 | and %o1, %o4, %o1 | |
299 | add %o1, 0x200, %o1 | |
300 | sta %g0, [%o1] ASI_M_FLUSH_PROBE | |
301 | 1: | |
302 | sub %o1, %o4, %o1 | |
303 | cmp %o1, %o2 | |
304 | blu,a 1b | |
305 | sta %g0, [%o1] ASI_M_FLUSH_PROBE | |
306 | hypersparc_flush_tlb_range_out: | |
307 | retl | |
308 | sta %g5, [%g1] ASI_M_MMUREGS | |
309 | ||
310 | hypersparc_flush_tlb_page: | |
961246b4 | 311 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
312 | mov SRMMU_CTX_REG, %g1 |
313 | ld [%o0 + AOFF_mm_context], %o3 | |
314 | andn %o1, (PAGE_SIZE - 1), %o1 | |
315 | #ifndef CONFIG_SMP | |
316 | cmp %o3, -1 | |
317 | be hypersparc_flush_tlb_page_out | |
318 | #endif | |
319 | lda [%g1] ASI_M_MMUREGS, %g5 | |
320 | sta %o3, [%g1] ASI_M_MMUREGS | |
321 | sta %g0, [%o1] ASI_M_FLUSH_PROBE | |
322 | hypersparc_flush_tlb_page_out: | |
323 | retl | |
324 | sta %g5, [%g1] ASI_M_MMUREGS | |
325 | ||
326 | __INIT | |
327 | ||
328 | /* High speed page clear/copy. */ | |
329 | hypersparc_bzero_1page: | |
330 | /* NOTE: This routine has to be shorter than 40insns --jj */ | |
331 | clr %g1 | |
332 | mov 32, %g2 | |
333 | mov 64, %g3 | |
334 | mov 96, %g4 | |
335 | mov 128, %g5 | |
336 | mov 160, %g7 | |
337 | mov 192, %o2 | |
338 | mov 224, %o3 | |
339 | mov 16, %o1 | |
340 | 1: | |
341 | stda %g0, [%o0 + %g0] ASI_M_BFILL | |
342 | stda %g0, [%o0 + %g2] ASI_M_BFILL | |
343 | stda %g0, [%o0 + %g3] ASI_M_BFILL | |
344 | stda %g0, [%o0 + %g4] ASI_M_BFILL | |
345 | stda %g0, [%o0 + %g5] ASI_M_BFILL | |
346 | stda %g0, [%o0 + %g7] ASI_M_BFILL | |
347 | stda %g0, [%o0 + %o2] ASI_M_BFILL | |
348 | stda %g0, [%o0 + %o3] ASI_M_BFILL | |
349 | subcc %o1, 1, %o1 | |
350 | bne 1b | |
351 | add %o0, 256, %o0 | |
352 | ||
353 | retl | |
354 | nop | |
355 | ||
356 | hypersparc_copy_1page: | |
357 | /* NOTE: This routine has to be shorter than 70insns --jj */ | |
358 | sub %o1, %o0, %o2 ! difference | |
359 | mov 16, %g1 | |
360 | 1: | |
361 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
362 | add %o0, 32, %o0 | |
363 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
364 | add %o0, 32, %o0 | |
365 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
366 | add %o0, 32, %o0 | |
367 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
368 | add %o0, 32, %o0 | |
369 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
370 | add %o0, 32, %o0 | |
371 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
372 | add %o0, 32, %o0 | |
373 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
374 | add %o0, 32, %o0 | |
375 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
376 | subcc %g1, 1, %g1 | |
377 | bne 1b | |
378 | add %o0, 32, %o0 | |
379 | ||
380 | retl | |
381 | nop | |
382 | ||
383 | .globl hypersparc_setup_blockops | |
384 | hypersparc_setup_blockops: | |
385 | sethi %hi(bzero_1page), %o0 | |
386 | or %o0, %lo(bzero_1page), %o0 | |
387 | sethi %hi(hypersparc_bzero_1page), %o1 | |
388 | or %o1, %lo(hypersparc_bzero_1page), %o1 | |
389 | sethi %hi(hypersparc_copy_1page), %o2 | |
390 | or %o2, %lo(hypersparc_copy_1page), %o2 | |
391 | ld [%o1], %o4 | |
392 | 1: | |
393 | add %o1, 4, %o1 | |
394 | st %o4, [%o0] | |
395 | add %o0, 4, %o0 | |
396 | cmp %o1, %o2 | |
397 | bne 1b | |
398 | ld [%o1], %o4 | |
399 | sethi %hi(__copy_1page), %o0 | |
400 | or %o0, %lo(__copy_1page), %o0 | |
401 | sethi %hi(hypersparc_setup_blockops), %o2 | |
402 | or %o2, %lo(hypersparc_setup_blockops), %o2 | |
403 | ld [%o1], %o4 | |
404 | 1: | |
405 | add %o1, 4, %o1 | |
406 | st %o4, [%o0] | |
407 | add %o0, 4, %o0 | |
408 | cmp %o1, %o2 | |
409 | bne 1b | |
410 | ld [%o1], %o4 | |
411 | sta %g0, [%g0] ASI_M_FLUSH_IWHOLE | |
412 | retl | |
413 | nop |