]> git.ipfire.org Git - thirdparty/linux.git/blob - arch/powerpc/kernel/entry_32.S
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
MAINTAINERS: Fix Hyperv vIOMMU driver file name
[thirdparty/linux.git] / arch / powerpc / kernel / entry_32.S
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * PowerPC version
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 *
13 * This file contains the system call entry code, context switch
14 * code, and exception/interrupt return code for PowerPC.
15 */
16
17 #include <linux/errno.h>
18 #include <linux/err.h>
19 #include <linux/sys.h>
20 #include <linux/threads.h>
21 #include <asm/reg.h>
22 #include <asm/page.h>
23 #include <asm/mmu.h>
24 #include <asm/cputable.h>
25 #include <asm/thread_info.h>
26 #include <asm/ppc_asm.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/unistd.h>
29 #include <asm/ptrace.h>
30 #include <asm/export.h>
31 #include <asm/asm-405.h>
32 #include <asm/feature-fixups.h>
33 #include <asm/barrier.h>
34 #include <asm/kup.h>
35 #include <asm/bug.h>
36
37 #include "head_32.h"
38
39 /*
40 * Align to 4k in order to ensure that all functions modyfing srr0/srr1
41 * fit into one page in order to not encounter a TLB miss between the
42 * modification of srr0/srr1 and the associated rfi.
43 */
44 .align 12
45
46 #ifdef CONFIG_BOOKE
47 .globl mcheck_transfer_to_handler
48 mcheck_transfer_to_handler:
49 mfspr r0,SPRN_DSRR0
50 stw r0,_DSRR0(r11)
51 mfspr r0,SPRN_DSRR1
52 stw r0,_DSRR1(r11)
53 /* fall through */
54
55 .globl debug_transfer_to_handler
56 debug_transfer_to_handler:
57 mfspr r0,SPRN_CSRR0
58 stw r0,_CSRR0(r11)
59 mfspr r0,SPRN_CSRR1
60 stw r0,_CSRR1(r11)
61 /* fall through */
62
63 .globl crit_transfer_to_handler
64 crit_transfer_to_handler:
65 #ifdef CONFIG_PPC_BOOK3E_MMU
66 mfspr r0,SPRN_MAS0
67 stw r0,MAS0(r11)
68 mfspr r0,SPRN_MAS1
69 stw r0,MAS1(r11)
70 mfspr r0,SPRN_MAS2
71 stw r0,MAS2(r11)
72 mfspr r0,SPRN_MAS3
73 stw r0,MAS3(r11)
74 mfspr r0,SPRN_MAS6
75 stw r0,MAS6(r11)
76 #ifdef CONFIG_PHYS_64BIT
77 mfspr r0,SPRN_MAS7
78 stw r0,MAS7(r11)
79 #endif /* CONFIG_PHYS_64BIT */
80 #endif /* CONFIG_PPC_BOOK3E_MMU */
81 #ifdef CONFIG_44x
82 mfspr r0,SPRN_MMUCR
83 stw r0,MMUCR(r11)
84 #endif
85 mfspr r0,SPRN_SRR0
86 stw r0,_SRR0(r11)
87 mfspr r0,SPRN_SRR1
88 stw r0,_SRR1(r11)
89
90 /* set the stack limit to the current stack */
91 mfspr r8,SPRN_SPRG_THREAD
92 lwz r0,KSP_LIMIT(r8)
93 stw r0,SAVED_KSP_LIMIT(r11)
94 rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
95 stw r0,KSP_LIMIT(r8)
96 /* fall through */
97 #endif
98
99 #ifdef CONFIG_40x
100 .globl crit_transfer_to_handler
101 crit_transfer_to_handler:
102 lwz r0,crit_r10@l(0)
103 stw r0,GPR10(r11)
104 lwz r0,crit_r11@l(0)
105 stw r0,GPR11(r11)
106 mfspr r0,SPRN_SRR0
107 stw r0,crit_srr0@l(0)
108 mfspr r0,SPRN_SRR1
109 stw r0,crit_srr1@l(0)
110
111 /* set the stack limit to the current stack */
112 mfspr r8,SPRN_SPRG_THREAD
113 lwz r0,KSP_LIMIT(r8)
114 stw r0,saved_ksp_limit@l(0)
115 rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
116 stw r0,KSP_LIMIT(r8)
117 /* fall through */
118 #endif
119
120 /*
121 * This code finishes saving the registers to the exception frame
122 * and jumps to the appropriate handler for the exception, turning
123 * on address translation.
124 * Note that we rely on the caller having set cr0.eq iff the exception
125 * occurred in kernel mode (i.e. MSR:PR = 0).
126 */
127 .globl transfer_to_handler_full
128 transfer_to_handler_full:
129 SAVE_NVGPRS(r11)
130 /* fall through */
131
132 .globl transfer_to_handler
133 transfer_to_handler:
134 stw r2,GPR2(r11)
135 stw r12,_NIP(r11)
136 stw r9,_MSR(r11)
137 andi. r2,r9,MSR_PR
138 mfctr r12
139 mfspr r2,SPRN_XER
140 stw r12,_CTR(r11)
141 stw r2,_XER(r11)
142 mfspr r12,SPRN_SPRG_THREAD
143 beq 2f /* if from user, fix up THREAD.regs */
144 addi r2, r12, -THREAD
145 addi r11,r1,STACK_FRAME_OVERHEAD
146 stw r11,PT_REGS(r12)
147 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
148 /* Check to see if the dbcr0 register is set up to debug. Use the
149 internal debug mode bit to do this. */
150 lwz r12,THREAD_DBCR0(r12)
151 andis. r12,r12,DBCR0_IDM@h
152 #endif
153 ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
154 #ifdef CONFIG_PPC_BOOK3S_32
155 kuep_lock r11, r12
156 #endif
157 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
158 beq+ 3f
159 /* From user and task is ptraced - load up global dbcr0 */
160 li r12,-1 /* clear all pending debug events */
161 mtspr SPRN_DBSR,r12
162 lis r11,global_dbcr0@ha
163 tophys(r11,r11)
164 addi r11,r11,global_dbcr0@l
165 #ifdef CONFIG_SMP
166 lwz r9,TASK_CPU(r2)
167 slwi r9,r9,3
168 add r11,r11,r9
169 #endif
170 lwz r12,0(r11)
171 mtspr SPRN_DBCR0,r12
172 lwz r12,4(r11)
173 addi r12,r12,-1
174 stw r12,4(r11)
175 #endif
176
177 b 3f
178
179 2: /* if from kernel, check interrupted DOZE/NAP mode and
180 * check for stack overflow
181 */
182 kuap_save_and_lock r11, r12, r9, r2, r0
183 addi r2, r12, -THREAD
184 lwz r9,KSP_LIMIT(r12)
185 cmplw r1,r9 /* if r1 <= ksp_limit */
186 ble- stack_ovf /* then the kernel stack overflowed */
187 5:
188 #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
189 lwz r12,TI_LOCAL_FLAGS(r2)
190 mtcrf 0x01,r12
191 bt- 31-TLF_NAPPING,4f
192 bt- 31-TLF_SLEEPING,7f
193 #endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */
194 .globl transfer_to_handler_cont
195 transfer_to_handler_cont:
196 3:
197 mflr r9
198 tovirt(r2, r2) /* set r2 to current */
199 lwz r11,0(r9) /* virtual address of handler */
200 lwz r9,4(r9) /* where to go when done */
201 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
202 mtspr SPRN_NRI, r0
203 #endif
204 #ifdef CONFIG_TRACE_IRQFLAGS
205 /*
206 * When tracing IRQ state (lockdep) we enable the MMU before we call
207 * the IRQ tracing functions as they might access vmalloc space or
208 * perform IOs for console output.
209 *
210 * To speed up the syscall path where interrupts stay on, let's check
211 * first if we are changing the MSR value at all.
212 */
213 tophys(r12, r1)
214 lwz r12,_MSR(r12)
215 andi. r12,r12,MSR_EE
216 bne 1f
217
218 /* MSR isn't changing, just transition directly */
219 #endif
220 mtspr SPRN_SRR0,r11
221 mtspr SPRN_SRR1,r10
222 mtlr r9
223 SYNC
224 RFI /* jump to handler, enable MMU */
225
226 #ifdef CONFIG_TRACE_IRQFLAGS
227 1: /* MSR is changing, re-enable MMU so we can notify lockdep. We need to
228 * keep interrupts disabled at this point otherwise we might risk
229 * taking an interrupt before we tell lockdep they are enabled.
230 */
231 lis r12,reenable_mmu@h
232 ori r12,r12,reenable_mmu@l
233 LOAD_MSR_KERNEL(r0, MSR_KERNEL)
234 mtspr SPRN_SRR0,r12
235 mtspr SPRN_SRR1,r0
236 SYNC
237 RFI
238
239 reenable_mmu:
240 /*
241 * We save a bunch of GPRs,
242 * r3 can be different from GPR3(r1) at this point, r9 and r11
243 * contains the old MSR and handler address respectively,
244 * r4 & r5 can contain page fault arguments that need to be passed
245 * along as well. r12, CCR, CTR, XER etc... are left clobbered as
246 * they aren't useful past this point (aren't syscall arguments),
247 * the rest is restored from the exception frame.
248 */
249
250 stwu r1,-32(r1)
251 stw r9,8(r1)
252 stw r11,12(r1)
253 stw r3,16(r1)
254 stw r4,20(r1)
255 stw r5,24(r1)
256
257 /* If we are disabling interrupts (normal case), simply log it with
258 * lockdep
259 */
260 1: bl trace_hardirqs_off
261 2: lwz r5,24(r1)
262 lwz r4,20(r1)
263 lwz r3,16(r1)
264 lwz r11,12(r1)
265 lwz r9,8(r1)
266 addi r1,r1,32
267 lwz r0,GPR0(r1)
268 lwz r6,GPR6(r1)
269 lwz r7,GPR7(r1)
270 lwz r8,GPR8(r1)
271 mtctr r11
272 mtlr r9
273 bctr /* jump to handler */
274 #endif /* CONFIG_TRACE_IRQFLAGS */
275
276 #if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
277 4: rlwinm r12,r12,0,~_TLF_NAPPING
278 stw r12,TI_LOCAL_FLAGS(r2)
279 b power_save_ppc32_restore
280
281 7: rlwinm r12,r12,0,~_TLF_SLEEPING
282 stw r12,TI_LOCAL_FLAGS(r2)
283 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
284 rlwinm r9,r9,0,~MSR_EE
285 lwz r12,_LINK(r11) /* and return to address in LR */
286 kuap_restore r11, r2, r3, r4, r5
287 b fast_exception_return
288 #endif
289
290 /*
291 * On kernel stack overflow, load up an initial stack pointer
292 * and call StackOverflow(regs), which should not return.
293 */
294 stack_ovf:
295 /* sometimes we use a statically-allocated stack, which is OK. */
296 lis r12,_end@h
297 ori r12,r12,_end@l
298 cmplw r1,r12
299 ble 5b /* r1 <= &_end is OK */
300 SAVE_NVGPRS(r11)
301 addi r3,r1,STACK_FRAME_OVERHEAD
302 lis r1,init_thread_union@ha
303 addi r1,r1,init_thread_union@l
304 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
305 lis r9,StackOverflow@ha
306 addi r9,r9,StackOverflow@l
307 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
308 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
309 mtspr SPRN_NRI, r0
310 #endif
311 mtspr SPRN_SRR0,r9
312 mtspr SPRN_SRR1,r10
313 SYNC
314 RFI
315
316 #ifdef CONFIG_TRACE_IRQFLAGS
317 trace_syscall_entry_irq_off:
318 /*
319 * Syscall shouldn't happen while interrupts are disabled,
320 * so let's do a warning here.
321 */
322 0: trap
323 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
324 bl trace_hardirqs_on
325
326 /* Now enable for real */
327 LOAD_MSR_KERNEL(r10, MSR_KERNEL | MSR_EE)
328 mtmsr r10
329
330 REST_GPR(0, r1)
331 REST_4GPRS(3, r1)
332 REST_2GPRS(7, r1)
333 b DoSyscall
334 #endif /* CONFIG_TRACE_IRQFLAGS */
335
336 .globl transfer_to_syscall
337 transfer_to_syscall:
338 #ifdef CONFIG_TRACE_IRQFLAGS
339 andi. r12,r9,MSR_EE
340 beq- trace_syscall_entry_irq_off
341 #endif /* CONFIG_TRACE_IRQFLAGS */
342
343 /*
344 * Handle a system call.
345 */
346 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
347 .stabs "entry_32.S",N_SO,0,0,0f
348 0:
349
350 _GLOBAL(DoSyscall)
351 stw r3,ORIG_GPR3(r1)
352 li r12,0
353 stw r12,RESULT(r1)
354 #ifdef CONFIG_TRACE_IRQFLAGS
355 /* Make sure interrupts are enabled */
356 mfmsr r11
357 andi. r12,r11,MSR_EE
358 /* We came in with interrupts disabled, we WARN and mark them enabled
359 * for lockdep now */
360 0: tweqi r12, 0
361 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
362 #endif /* CONFIG_TRACE_IRQFLAGS */
363 lwz r11,TI_FLAGS(r2)
364 andi. r11,r11,_TIF_SYSCALL_DOTRACE
365 bne- syscall_dotrace
366 syscall_dotrace_cont:
367 cmplwi 0,r0,NR_syscalls
368 lis r10,sys_call_table@h
369 ori r10,r10,sys_call_table@l
370 slwi r0,r0,2
371 bge- 66f
372
373 barrier_nospec_asm
374 /*
375 * Prevent the load of the handler below (based on the user-passed
376 * system call number) being speculatively executed until the test
377 * against NR_syscalls and branch to .66f above has
378 * committed.
379 */
380
381 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
382 mtlr r10
383 addi r9,r1,STACK_FRAME_OVERHEAD
384 PPC440EP_ERR42
385 blrl /* Call handler */
386 .globl ret_from_syscall
387 ret_from_syscall:
388 #ifdef CONFIG_DEBUG_RSEQ
389 /* Check whether the syscall is issued inside a restartable sequence */
390 stw r3,GPR3(r1)
391 addi r3,r1,STACK_FRAME_OVERHEAD
392 bl rseq_syscall
393 lwz r3,GPR3(r1)
394 #endif
395 mr r6,r3
396 /* disable interrupts so current_thread_info()->flags can't change */
397 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
398 /* Note: We don't bother telling lockdep about it */
399 SYNC
400 MTMSRD(r10)
401 lwz r9,TI_FLAGS(r2)
402 li r8,-MAX_ERRNO
403 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
404 bne- syscall_exit_work
405 cmplw 0,r3,r8
406 blt+ syscall_exit_cont
407 lwz r11,_CCR(r1) /* Load CR */
408 neg r3,r3
409 oris r11,r11,0x1000 /* Set SO bit in CR */
410 stw r11,_CCR(r1)
411 syscall_exit_cont:
412 lwz r8,_MSR(r1)
413 #ifdef CONFIG_TRACE_IRQFLAGS
414 /* If we are going to return from the syscall with interrupts
415 * off, we trace that here. It shouldn't normally happen.
416 */
417 andi. r10,r8,MSR_EE
418 bne+ 1f
419 stw r3,GPR3(r1)
420 bl trace_hardirqs_off
421 lwz r3,GPR3(r1)
422 1:
423 #endif /* CONFIG_TRACE_IRQFLAGS */
424 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
425 /* If the process has its own DBCR0 value, load it up. The internal
426 debug mode bit tells us that dbcr0 should be loaded. */
427 lwz r0,THREAD+THREAD_DBCR0(r2)
428 andis. r10,r0,DBCR0_IDM@h
429 bnel- load_dbcr0
430 #endif
431 #ifdef CONFIG_44x
432 BEGIN_MMU_FTR_SECTION
433 lis r4,icache_44x_need_flush@ha
434 lwz r5,icache_44x_need_flush@l(r4)
435 cmplwi cr0,r5,0
436 bne- 2f
437 1:
438 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
439 #endif /* CONFIG_44x */
440 BEGIN_FTR_SECTION
441 lwarx r7,0,r1
442 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
443 stwcx. r0,0,r1 /* to clear the reservation */
444 ACCOUNT_CPU_USER_EXIT(r2, r5, r7)
445 #ifdef CONFIG_PPC_BOOK3S_32
446 kuep_unlock r5, r7
447 #endif
448 kuap_check r2, r4
449 lwz r4,_LINK(r1)
450 lwz r5,_CCR(r1)
451 mtlr r4
452 mtcr r5
453 lwz r7,_NIP(r1)
454 lwz r2,GPR2(r1)
455 lwz r1,GPR1(r1)
456 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
457 mtspr SPRN_NRI, r0
458 #endif
459 mtspr SPRN_SRR0,r7
460 mtspr SPRN_SRR1,r8
461 SYNC
462 RFI
463 #ifdef CONFIG_44x
464 2: li r7,0
465 iccci r0,r0
466 stw r7,icache_44x_need_flush@l(r4)
467 b 1b
468 #endif /* CONFIG_44x */
469
470 66: li r3,-ENOSYS
471 b ret_from_syscall
472
473 .globl ret_from_fork
474 ret_from_fork:
475 REST_NVGPRS(r1)
476 bl schedule_tail
477 li r3,0
478 b ret_from_syscall
479
480 .globl ret_from_kernel_thread
481 ret_from_kernel_thread:
482 REST_NVGPRS(r1)
483 bl schedule_tail
484 mtlr r14
485 mr r3,r15
486 PPC440EP_ERR42
487 blrl
488 li r3,0
489 b ret_from_syscall
490
491 /* Traced system call support */
492 syscall_dotrace:
493 SAVE_NVGPRS(r1)
494 li r0,0xc00
495 stw r0,_TRAP(r1)
496 addi r3,r1,STACK_FRAME_OVERHEAD
497 bl do_syscall_trace_enter
498 /*
499 * Restore argument registers possibly just changed.
500 * We use the return value of do_syscall_trace_enter
501 * for call number to look up in the table (r0).
502 */
503 mr r0,r3
504 lwz r3,GPR3(r1)
505 lwz r4,GPR4(r1)
506 lwz r5,GPR5(r1)
507 lwz r6,GPR6(r1)
508 lwz r7,GPR7(r1)
509 lwz r8,GPR8(r1)
510 REST_NVGPRS(r1)
511
512 cmplwi r0,NR_syscalls
513 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
514 bge- ret_from_syscall
515 b syscall_dotrace_cont
516
517 syscall_exit_work:
518 andi. r0,r9,_TIF_RESTOREALL
519 beq+ 0f
520 REST_NVGPRS(r1)
521 b 2f
522 0: cmplw 0,r3,r8
523 blt+ 1f
524 andi. r0,r9,_TIF_NOERROR
525 bne- 1f
526 lwz r11,_CCR(r1) /* Load CR */
527 neg r3,r3
528 oris r11,r11,0x1000 /* Set SO bit in CR */
529 stw r11,_CCR(r1)
530
531 1: stw r6,RESULT(r1) /* Save result */
532 stw r3,GPR3(r1) /* Update return value */
533 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
534 beq 4f
535
536 /* Clear per-syscall TIF flags if any are set. */
537
538 li r11,_TIF_PERSYSCALL_MASK
539 addi r12,r2,TI_FLAGS
540 3: lwarx r8,0,r12
541 andc r8,r8,r11
542 #ifdef CONFIG_IBM405_ERR77
543 dcbt 0,r12
544 #endif
545 stwcx. r8,0,r12
546 bne- 3b
547
548 4: /* Anything which requires enabling interrupts? */
549 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
550 beq ret_from_except
551
552 /* Re-enable interrupts. There is no need to trace that with
553 * lockdep as we are supposed to have IRQs on at this point
554 */
555 ori r10,r10,MSR_EE
556 SYNC
557 MTMSRD(r10)
558
559 /* Save NVGPRS if they're not saved already */
560 lwz r4,_TRAP(r1)
561 andi. r4,r4,1
562 beq 5f
563 SAVE_NVGPRS(r1)
564 li r4,0xc00
565 stw r4,_TRAP(r1)
566 5:
567 addi r3,r1,STACK_FRAME_OVERHEAD
568 bl do_syscall_trace_leave
569 b ret_from_except_full
570
571 /*
572 * The fork/clone functions need to copy the full register set into
573 * the child process. Therefore we need to save all the nonvolatile
574 * registers (r13 - r31) before calling the C code.
575 */
576 .globl ppc_fork
577 ppc_fork:
578 SAVE_NVGPRS(r1)
579 lwz r0,_TRAP(r1)
580 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
581 stw r0,_TRAP(r1) /* register set saved */
582 b sys_fork
583
584 .globl ppc_vfork
585 ppc_vfork:
586 SAVE_NVGPRS(r1)
587 lwz r0,_TRAP(r1)
588 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
589 stw r0,_TRAP(r1) /* register set saved */
590 b sys_vfork
591
592 .globl ppc_clone
593 ppc_clone:
594 SAVE_NVGPRS(r1)
595 lwz r0,_TRAP(r1)
596 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
597 stw r0,_TRAP(r1) /* register set saved */
598 b sys_clone
599
600 .globl ppc_swapcontext
601 ppc_swapcontext:
602 SAVE_NVGPRS(r1)
603 lwz r0,_TRAP(r1)
604 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
605 stw r0,_TRAP(r1) /* register set saved */
606 b sys_swapcontext
607
608 /*
609 * Top-level page fault handling.
610 * This is in assembler because if do_page_fault tells us that
611 * it is a bad kernel page fault, we want to save the non-volatile
612 * registers before calling bad_page_fault.
613 */
614 .globl handle_page_fault
615 handle_page_fault:
616 stw r4,_DAR(r1)
617 addi r3,r1,STACK_FRAME_OVERHEAD
618 #ifdef CONFIG_PPC_BOOK3S_32
619 andis. r0,r5,DSISR_DABRMATCH@h
620 bne- handle_dabr_fault
621 #endif
622 bl do_page_fault
623 cmpwi r3,0
624 beq+ ret_from_except
625 SAVE_NVGPRS(r1)
626 lwz r0,_TRAP(r1)
627 clrrwi r0,r0,1
628 stw r0,_TRAP(r1)
629 mr r5,r3
630 addi r3,r1,STACK_FRAME_OVERHEAD
631 lwz r4,_DAR(r1)
632 bl bad_page_fault
633 b ret_from_except_full
634
635 #ifdef CONFIG_PPC_BOOK3S_32
636 /* We have a data breakpoint exception - handle it */
637 handle_dabr_fault:
638 SAVE_NVGPRS(r1)
639 lwz r0,_TRAP(r1)
640 clrrwi r0,r0,1
641 stw r0,_TRAP(r1)
642 bl do_break
643 b ret_from_except_full
644 #endif
645
646 /*
647 * This routine switches between two different tasks. The process
648 * state of one is saved on its kernel stack. Then the state
649 * of the other is restored from its kernel stack. The memory
650 * management hardware is updated to the second process's state.
651 * Finally, we can return to the second process.
652 * On entry, r3 points to the THREAD for the current task, r4
653 * points to the THREAD for the new task.
654 *
655 * This routine is always called with interrupts disabled.
656 *
657 * Note: there are two ways to get to the "going out" portion
658 * of this code; either by coming in via the entry (_switch)
659 * or via "fork" which must set up an environment equivalent
660 * to the "_switch" path. If you change this , you'll have to
661 * change the fork code also.
662 *
663 * The code which creates the new task context is in 'copy_thread'
664 * in arch/ppc/kernel/process.c
665 */
666 _GLOBAL(_switch)
667 stwu r1,-INT_FRAME_SIZE(r1)
668 mflr r0
669 stw r0,INT_FRAME_SIZE+4(r1)
670 /* r3-r12 are caller saved -- Cort */
671 SAVE_NVGPRS(r1)
672 stw r0,_NIP(r1) /* Return to switch caller */
673 mfmsr r11
674 li r0,MSR_FP /* Disable floating-point */
675 #ifdef CONFIG_ALTIVEC
676 BEGIN_FTR_SECTION
677 oris r0,r0,MSR_VEC@h /* Disable altivec */
678 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
679 stw r12,THREAD+THREAD_VRSAVE(r2)
680 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
681 #endif /* CONFIG_ALTIVEC */
682 #ifdef CONFIG_SPE
683 BEGIN_FTR_SECTION
684 oris r0,r0,MSR_SPE@h /* Disable SPE */
685 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
686 stw r12,THREAD+THREAD_SPEFSCR(r2)
687 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
688 #endif /* CONFIG_SPE */
689 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
690 beq+ 1f
691 andc r11,r11,r0
692 MTMSRD(r11)
693 isync
694 1: stw r11,_MSR(r1)
695 mfcr r10
696 stw r10,_CCR(r1)
697 stw r1,KSP(r3) /* Set old stack pointer */
698
699 kuap_check r2, r4
700 #ifdef CONFIG_SMP
701 /* We need a sync somewhere here to make sure that if the
702 * previous task gets rescheduled on another CPU, it sees all
703 * stores it has performed on this one.
704 */
705 sync
706 #endif /* CONFIG_SMP */
707
708 tophys(r0,r4)
709 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
710 lwz r1,KSP(r4) /* Load new stack pointer */
711
712 /* save the old current 'last' for return value */
713 mr r3,r2
714 addi r2,r4,-THREAD /* Update current */
715
716 #ifdef CONFIG_ALTIVEC
717 BEGIN_FTR_SECTION
718 lwz r0,THREAD+THREAD_VRSAVE(r2)
719 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
720 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
721 #endif /* CONFIG_ALTIVEC */
722 #ifdef CONFIG_SPE
723 BEGIN_FTR_SECTION
724 lwz r0,THREAD+THREAD_SPEFSCR(r2)
725 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
726 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
727 #endif /* CONFIG_SPE */
728
729 lwz r0,_CCR(r1)
730 mtcrf 0xFF,r0
731 /* r3-r12 are destroyed -- Cort */
732 REST_NVGPRS(r1)
733
734 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
735 mtlr r4
736 addi r1,r1,INT_FRAME_SIZE
737 blr
738
739 .globl fast_exception_return
740 fast_exception_return:
741 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
742 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
743 beq 1f /* if not, we've got problems */
744 #endif
745
746 2: REST_4GPRS(3, r11)
747 lwz r10,_CCR(r11)
748 REST_GPR(1, r11)
749 mtcr r10
750 lwz r10,_LINK(r11)
751 mtlr r10
752 /* Clear the exception_marker on the stack to avoid confusing stacktrace */
753 li r10, 0
754 stw r10, 8(r11)
755 REST_GPR(10, r11)
756 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
757 mtspr SPRN_NRI, r0
758 #endif
759 mtspr SPRN_SRR1,r9
760 mtspr SPRN_SRR0,r12
761 REST_GPR(9, r11)
762 REST_GPR(12, r11)
763 lwz r11,GPR11(r11)
764 SYNC
765 RFI
766
767 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
768 /* check if the exception happened in a restartable section */
769 1: lis r3,exc_exit_restart_end@ha
770 addi r3,r3,exc_exit_restart_end@l
771 cmplw r12,r3
772 bge 3f
773 lis r4,exc_exit_restart@ha
774 addi r4,r4,exc_exit_restart@l
775 cmplw r12,r4
776 blt 3f
777 lis r3,fee_restarts@ha
778 tophys(r3,r3)
779 lwz r5,fee_restarts@l(r3)
780 addi r5,r5,1
781 stw r5,fee_restarts@l(r3)
782 mr r12,r4 /* restart at exc_exit_restart */
783 b 2b
784
785 .section .bss
786 .align 2
787 fee_restarts:
788 .space 4
789 .previous
790
791 /* aargh, a nonrecoverable interrupt, panic */
792 /* aargh, we don't know which trap this is */
793 /* but the 601 doesn't implement the RI bit, so assume it's OK */
794 3:
795 BEGIN_FTR_SECTION
796 b 2b
797 END_FTR_SECTION_IFSET(CPU_FTR_601)
798 li r10,-1
799 stw r10,_TRAP(r11)
800 addi r3,r1,STACK_FRAME_OVERHEAD
801 lis r10,MSR_KERNEL@h
802 ori r10,r10,MSR_KERNEL@l
803 bl transfer_to_handler_full
804 .long unrecoverable_exception
805 .long ret_from_except
806 #endif
807
808 .globl ret_from_except_full
809 ret_from_except_full:
810 REST_NVGPRS(r1)
811 /* fall through */
812
813 .globl ret_from_except
814 ret_from_except:
815 /* Hard-disable interrupts so that current_thread_info()->flags
816 * can't change between when we test it and when we return
817 * from the interrupt. */
818 /* Note: We don't bother telling lockdep about it */
819 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
820 SYNC /* Some chip revs have problems here... */
821 MTMSRD(r10) /* disable interrupts */
822
823 lwz r3,_MSR(r1) /* Returning to user mode? */
824 andi. r0,r3,MSR_PR
825 beq resume_kernel
826
827 user_exc_return: /* r10 contains MSR_KERNEL here */
828 /* Check current_thread_info()->flags */
829 lwz r9,TI_FLAGS(r2)
830 andi. r0,r9,_TIF_USER_WORK_MASK
831 bne do_work
832
833 restore_user:
834 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
835 /* Check whether this process has its own DBCR0 value. The internal
836 debug mode bit tells us that dbcr0 should be loaded. */
837 lwz r0,THREAD+THREAD_DBCR0(r2)
838 andis. r10,r0,DBCR0_IDM@h
839 bnel- load_dbcr0
840 #endif
841 ACCOUNT_CPU_USER_EXIT(r2, r10, r11)
842 #ifdef CONFIG_PPC_BOOK3S_32
843 kuep_unlock r10, r11
844 #endif
845
846 b restore
847
848 /* N.B. the only way to get here is from the beq following ret_from_except. */
849 resume_kernel:
850 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
851 lwz r8,TI_FLAGS(r2)
852 andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
853 beq+ 1f
854
855 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
856
857 lwz r3,GPR1(r1)
858 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
859 mr r4,r1 /* src: current exception frame */
860 mr r1,r3 /* Reroute the trampoline frame to r1 */
861
862 /* Copy from the original to the trampoline. */
863 li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
864 li r6,0 /* start offset: 0 */
865 mtctr r5
866 2: lwzx r0,r6,r4
867 stwx r0,r6,r3
868 addi r6,r6,4
869 bdnz 2b
870
871 /* Do real store operation to complete stwu */
872 lwz r5,GPR1(r1)
873 stw r8,0(r5)
874
875 /* Clear _TIF_EMULATE_STACK_STORE flag */
876 lis r11,_TIF_EMULATE_STACK_STORE@h
877 addi r5,r2,TI_FLAGS
878 0: lwarx r8,0,r5
879 andc r8,r8,r11
880 #ifdef CONFIG_IBM405_ERR77
881 dcbt 0,r5
882 #endif
883 stwcx. r8,0,r5
884 bne- 0b
885 1:
886
887 #ifdef CONFIG_PREEMPT
888 /* check current_thread_info->preempt_count */
889 lwz r0,TI_PREEMPT(r2)
890 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
891 bne restore_kuap
892 andi. r8,r8,_TIF_NEED_RESCHED
893 beq+ restore_kuap
894 lwz r3,_MSR(r1)
895 andi. r0,r3,MSR_EE /* interrupts off? */
896 beq restore_kuap /* don't schedule if so */
897 #ifdef CONFIG_TRACE_IRQFLAGS
898 /* Lockdep thinks irqs are enabled, we need to call
899 * preempt_schedule_irq with IRQs off, so we inform lockdep
900 * now that we -did- turn them off already
901 */
902 bl trace_hardirqs_off
903 #endif
904 bl preempt_schedule_irq
905 #ifdef CONFIG_TRACE_IRQFLAGS
906 /* And now, to properly rebalance the above, we tell lockdep they
907 * are being turned back on, which will happen when we return
908 */
909 bl trace_hardirqs_on
910 #endif
911 #endif /* CONFIG_PREEMPT */
912 restore_kuap:
913 kuap_restore r1, r2, r9, r10, r0
914
915 /* interrupts are hard-disabled at this point */
916 restore:
917 #ifdef CONFIG_44x
918 BEGIN_MMU_FTR_SECTION
919 b 1f
920 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
921 lis r4,icache_44x_need_flush@ha
922 lwz r5,icache_44x_need_flush@l(r4)
923 cmplwi cr0,r5,0
924 beq+ 1f
925 li r6,0
926 iccci r0,r0
927 stw r6,icache_44x_need_flush@l(r4)
928 1:
929 #endif /* CONFIG_44x */
930
931 lwz r9,_MSR(r1)
932 #ifdef CONFIG_TRACE_IRQFLAGS
933 /* Lockdep doesn't know about the fact that IRQs are temporarily turned
934 * off in this assembly code while peeking at TI_FLAGS() and such. However
935 * we need to inform it if the exception turned interrupts off, and we
936 * are about to trun them back on.
937 */
938 andi. r10,r9,MSR_EE
939 beq 1f
940 stwu r1,-32(r1)
941 mflr r0
942 stw r0,4(r1)
943 bl trace_hardirqs_on
944 addi r1, r1, 32
945 lwz r9,_MSR(r1)
946 1:
947 #endif /* CONFIG_TRACE_IRQFLAGS */
948
949 lwz r0,GPR0(r1)
950 lwz r2,GPR2(r1)
951 REST_4GPRS(3, r1)
952 REST_2GPRS(7, r1)
953
954 lwz r10,_XER(r1)
955 lwz r11,_CTR(r1)
956 mtspr SPRN_XER,r10
957 mtctr r11
958
959 PPC405_ERR77(0,r1)
960 BEGIN_FTR_SECTION
961 lwarx r11,0,r1
962 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
963 stwcx. r0,0,r1 /* to clear the reservation */
964
965 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
966 andi. r10,r9,MSR_RI /* check if this exception occurred */
967 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
968
969 lwz r10,_CCR(r1)
970 lwz r11,_LINK(r1)
971 mtcrf 0xFF,r10
972 mtlr r11
973
974 /* Clear the exception_marker on the stack to avoid confusing stacktrace */
975 li r10, 0
976 stw r10, 8(r1)
977 /*
978 * Once we put values in SRR0 and SRR1, we are in a state
979 * where exceptions are not recoverable, since taking an
980 * exception will trash SRR0 and SRR1. Therefore we clear the
981 * MSR:RI bit to indicate this. If we do take an exception,
982 * we can't return to the point of the exception but we
983 * can restart the exception exit path at the label
984 * exc_exit_restart below. -- paulus
985 */
986 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
987 SYNC
988 MTMSRD(r10) /* clear the RI bit */
989 .globl exc_exit_restart
990 exc_exit_restart:
991 lwz r12,_NIP(r1)
992 mtspr SPRN_SRR0,r12
993 mtspr SPRN_SRR1,r9
994 REST_4GPRS(9, r1)
995 lwz r1,GPR1(r1)
996 .globl exc_exit_restart_end
997 exc_exit_restart_end:
998 SYNC
999 RFI
1000
1001 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
1002 /*
1003 * This is a bit different on 4xx/Book-E because it doesn't have
1004 * the RI bit in the MSR.
1005 * The TLB miss handler checks if we have interrupted
1006 * the exception exit path and restarts it if so
1007 * (well maybe one day it will... :).
1008 */
1009 lwz r11,_LINK(r1)
1010 mtlr r11
1011 lwz r10,_CCR(r1)
1012 mtcrf 0xff,r10
1013 /* Clear the exception_marker on the stack to avoid confusing stacktrace */
1014 li r10, 0
1015 stw r10, 8(r1)
1016 REST_2GPRS(9, r1)
1017 .globl exc_exit_restart
1018 exc_exit_restart:
1019 lwz r11,_NIP(r1)
1020 lwz r12,_MSR(r1)
1021 exc_exit_start:
1022 mtspr SPRN_SRR0,r11
1023 mtspr SPRN_SRR1,r12
1024 REST_2GPRS(11, r1)
1025 lwz r1,GPR1(r1)
1026 .globl exc_exit_restart_end
1027 exc_exit_restart_end:
1028 PPC405_ERR77_SYNC
1029 rfi
1030 b . /* prevent prefetch past rfi */
1031
1032 /*
1033 * Returning from a critical interrupt in user mode doesn't need
1034 * to be any different from a normal exception. For a critical
1035 * interrupt in the kernel, we just return (without checking for
1036 * preemption) since the interrupt may have happened at some crucial
1037 * place (e.g. inside the TLB miss handler), and because we will be
1038 * running with r1 pointing into critical_stack, not the current
1039 * process's kernel stack (and therefore current_thread_info() will
1040 * give the wrong answer).
1041 * We have to restore various SPRs that may have been in use at the
1042 * time of the critical interrupt.
1043 *
1044 */
1045 #ifdef CONFIG_40x
1046 #define PPC_40x_TURN_OFF_MSR_DR \
1047 /* avoid any possible TLB misses here by turning off MSR.DR, we \
1048 * assume the instructions here are mapped by a pinned TLB entry */ \
1049 li r10,MSR_IR; \
1050 mtmsr r10; \
1051 isync; \
1052 tophys(r1, r1);
1053 #else
1054 #define PPC_40x_TURN_OFF_MSR_DR
1055 #endif
1056
1057 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
1058 REST_NVGPRS(r1); \
1059 lwz r3,_MSR(r1); \
1060 andi. r3,r3,MSR_PR; \
1061 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \
1062 bne user_exc_return; \
1063 lwz r0,GPR0(r1); \
1064 lwz r2,GPR2(r1); \
1065 REST_4GPRS(3, r1); \
1066 REST_2GPRS(7, r1); \
1067 lwz r10,_XER(r1); \
1068 lwz r11,_CTR(r1); \
1069 mtspr SPRN_XER,r10; \
1070 mtctr r11; \
1071 PPC405_ERR77(0,r1); \
1072 stwcx. r0,0,r1; /* to clear the reservation */ \
1073 lwz r11,_LINK(r1); \
1074 mtlr r11; \
1075 lwz r10,_CCR(r1); \
1076 mtcrf 0xff,r10; \
1077 PPC_40x_TURN_OFF_MSR_DR; \
1078 lwz r9,_DEAR(r1); \
1079 lwz r10,_ESR(r1); \
1080 mtspr SPRN_DEAR,r9; \
1081 mtspr SPRN_ESR,r10; \
1082 lwz r11,_NIP(r1); \
1083 lwz r12,_MSR(r1); \
1084 mtspr exc_lvl_srr0,r11; \
1085 mtspr exc_lvl_srr1,r12; \
1086 lwz r9,GPR9(r1); \
1087 lwz r12,GPR12(r1); \
1088 lwz r10,GPR10(r1); \
1089 lwz r11,GPR11(r1); \
1090 lwz r1,GPR1(r1); \
1091 PPC405_ERR77_SYNC; \
1092 exc_lvl_rfi; \
1093 b .; /* prevent prefetch past exc_lvl_rfi */
1094
1095 #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
1096 lwz r9,_##exc_lvl_srr0(r1); \
1097 lwz r10,_##exc_lvl_srr1(r1); \
1098 mtspr SPRN_##exc_lvl_srr0,r9; \
1099 mtspr SPRN_##exc_lvl_srr1,r10;
1100
1101 #if defined(CONFIG_PPC_BOOK3E_MMU)
1102 #ifdef CONFIG_PHYS_64BIT
1103 #define RESTORE_MAS7 \
1104 lwz r11,MAS7(r1); \
1105 mtspr SPRN_MAS7,r11;
1106 #else
1107 #define RESTORE_MAS7
1108 #endif /* CONFIG_PHYS_64BIT */
1109 #define RESTORE_MMU_REGS \
1110 lwz r9,MAS0(r1); \
1111 lwz r10,MAS1(r1); \
1112 lwz r11,MAS2(r1); \
1113 mtspr SPRN_MAS0,r9; \
1114 lwz r9,MAS3(r1); \
1115 mtspr SPRN_MAS1,r10; \
1116 lwz r10,MAS6(r1); \
1117 mtspr SPRN_MAS2,r11; \
1118 mtspr SPRN_MAS3,r9; \
1119 mtspr SPRN_MAS6,r10; \
1120 RESTORE_MAS7;
1121 #elif defined(CONFIG_44x)
1122 #define RESTORE_MMU_REGS \
1123 lwz r9,MMUCR(r1); \
1124 mtspr SPRN_MMUCR,r9;
1125 #else
1126 #define RESTORE_MMU_REGS
1127 #endif
1128
1129 #ifdef CONFIG_40x
1130 .globl ret_from_crit_exc
1131 ret_from_crit_exc:
1132 mfspr r9,SPRN_SPRG_THREAD
1133 lis r10,saved_ksp_limit@ha;
1134 lwz r10,saved_ksp_limit@l(r10);
1135 tovirt(r9,r9);
1136 stw r10,KSP_LIMIT(r9)
1137 lis r9,crit_srr0@ha;
1138 lwz r9,crit_srr0@l(r9);
1139 lis r10,crit_srr1@ha;
1140 lwz r10,crit_srr1@l(r10);
1141 mtspr SPRN_SRR0,r9;
1142 mtspr SPRN_SRR1,r10;
1143 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1144 #endif /* CONFIG_40x */
1145
1146 #ifdef CONFIG_BOOKE
1147 .globl ret_from_crit_exc
1148 ret_from_crit_exc:
1149 mfspr r9,SPRN_SPRG_THREAD
1150 lwz r10,SAVED_KSP_LIMIT(r1)
1151 stw r10,KSP_LIMIT(r9)
1152 RESTORE_xSRR(SRR0,SRR1);
1153 RESTORE_MMU_REGS;
1154 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1155
1156 .globl ret_from_debug_exc
1157 ret_from_debug_exc:
1158 mfspr r9,SPRN_SPRG_THREAD
1159 lwz r10,SAVED_KSP_LIMIT(r1)
1160 stw r10,KSP_LIMIT(r9)
1161 RESTORE_xSRR(SRR0,SRR1);
1162 RESTORE_xSRR(CSRR0,CSRR1);
1163 RESTORE_MMU_REGS;
1164 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
1165
1166 .globl ret_from_mcheck_exc
1167 ret_from_mcheck_exc:
1168 mfspr r9,SPRN_SPRG_THREAD
1169 lwz r10,SAVED_KSP_LIMIT(r1)
1170 stw r10,KSP_LIMIT(r9)
1171 RESTORE_xSRR(SRR0,SRR1);
1172 RESTORE_xSRR(CSRR0,CSRR1);
1173 RESTORE_xSRR(DSRR0,DSRR1);
1174 RESTORE_MMU_REGS;
1175 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
1176 #endif /* CONFIG_BOOKE */
1177
1178 /*
1179 * Load the DBCR0 value for a task that is being ptraced,
1180 * having first saved away the global DBCR0. Note that r0
1181 * has the dbcr0 value to set upon entry to this.
1182 */
1183 load_dbcr0:
1184 mfmsr r10 /* first disable debug exceptions */
1185 rlwinm r10,r10,0,~MSR_DE
1186 mtmsr r10
1187 isync
1188 mfspr r10,SPRN_DBCR0
1189 lis r11,global_dbcr0@ha
1190 addi r11,r11,global_dbcr0@l
1191 #ifdef CONFIG_SMP
1192 lwz r9,TASK_CPU(r2)
1193 slwi r9,r9,3
1194 add r11,r11,r9
1195 #endif
1196 stw r10,0(r11)
1197 mtspr SPRN_DBCR0,r0
1198 lwz r10,4(r11)
1199 addi r10,r10,1
1200 stw r10,4(r11)
1201 li r11,-1
1202 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
1203 blr
1204
1205 .section .bss
1206 .align 4
1207 .global global_dbcr0
1208 global_dbcr0:
1209 .space 8*NR_CPUS
1210 .previous
1211 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
1212
1213 do_work: /* r10 contains MSR_KERNEL here */
1214 andi. r0,r9,_TIF_NEED_RESCHED
1215 beq do_user_signal
1216
1217 do_resched: /* r10 contains MSR_KERNEL here */
1218 #ifdef CONFIG_TRACE_IRQFLAGS
1219 bl trace_hardirqs_on
1220 mfmsr r10
1221 #endif
1222 ori r10,r10,MSR_EE
1223 SYNC
1224 MTMSRD(r10) /* hard-enable interrupts */
1225 bl schedule
1226 recheck:
1227 /* Note: And we don't tell it we are disabling them again
1228 * neither. Those disable/enable cycles used to peek at
1229 * TI_FLAGS aren't advertised.
1230 */
1231 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
1232 SYNC
1233 MTMSRD(r10) /* disable interrupts */
1234 lwz r9,TI_FLAGS(r2)
1235 andi. r0,r9,_TIF_NEED_RESCHED
1236 bne- do_resched
1237 andi. r0,r9,_TIF_USER_WORK_MASK
1238 beq restore_user
1239 do_user_signal: /* r10 contains MSR_KERNEL here */
1240 ori r10,r10,MSR_EE
1241 SYNC
1242 MTMSRD(r10) /* hard-enable interrupts */
1243 /* save r13-r31 in the exception frame, if not already done */
1244 lwz r3,_TRAP(r1)
1245 andi. r0,r3,1
1246 beq 2f
1247 SAVE_NVGPRS(r1)
1248 rlwinm r3,r3,0,0,30
1249 stw r3,_TRAP(r1)
1250 2: addi r3,r1,STACK_FRAME_OVERHEAD
1251 mr r4,r9
1252 bl do_notify_resume
1253 REST_NVGPRS(r1)
1254 b recheck
1255
1256 /*
1257 * We come here when we are at the end of handling an exception
1258 * that occurred at a place where taking an exception will lose
1259 * state information, such as the contents of SRR0 and SRR1.
1260 */
1261 nonrecoverable:
1262 lis r10,exc_exit_restart_end@ha
1263 addi r10,r10,exc_exit_restart_end@l
1264 cmplw r12,r10
1265 bge 3f
1266 lis r11,exc_exit_restart@ha
1267 addi r11,r11,exc_exit_restart@l
1268 cmplw r12,r11
1269 blt 3f
1270 lis r10,ee_restarts@ha
1271 lwz r12,ee_restarts@l(r10)
1272 addi r12,r12,1
1273 stw r12,ee_restarts@l(r10)
1274 mr r12,r11 /* restart at exc_exit_restart */
1275 blr
1276 3: /* OK, we can't recover, kill this process */
1277 /* but the 601 doesn't implement the RI bit, so assume it's OK */
1278 BEGIN_FTR_SECTION
1279 blr
1280 END_FTR_SECTION_IFSET(CPU_FTR_601)
1281 lwz r3,_TRAP(r1)
1282 andi. r0,r3,1
1283 beq 5f
1284 SAVE_NVGPRS(r1)
1285 rlwinm r3,r3,0,0,30
1286 stw r3,_TRAP(r1)
1287 5: mfspr r2,SPRN_SPRG_THREAD
1288 addi r2,r2,-THREAD
1289 tovirt(r2,r2) /* set back r2 to current */
1290 4: addi r3,r1,STACK_FRAME_OVERHEAD
1291 bl unrecoverable_exception
1292 /* shouldn't return */
1293 b 4b
1294
1295 .section .bss
1296 .align 2
1297 ee_restarts:
1298 .space 4
1299 .previous
1300
1301 /*
1302 * PROM code for specific machines follows. Put it
1303 * here so it's easy to add arch-specific sections later.
1304 * -- Cort
1305 */
1306 #ifdef CONFIG_PPC_RTAS
1307 /*
1308 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1309 * called with the MMU off.
1310 */
1311 _GLOBAL(enter_rtas)
1312 stwu r1,-INT_FRAME_SIZE(r1)
1313 mflr r0
1314 stw r0,INT_FRAME_SIZE+4(r1)
1315 LOAD_REG_ADDR(r4, rtas)
1316 lis r6,1f@ha /* physical return address for rtas */
1317 addi r6,r6,1f@l
1318 tophys(r6,r6)
1319 tophys(r7,r1)
1320 lwz r8,RTASENTRY(r4)
1321 lwz r4,RTASBASE(r4)
1322 mfmsr r9
1323 stw r9,8(r1)
1324 LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1325 SYNC /* disable interrupts so SRR0/1 */
1326 MTMSRD(r0) /* don't get trashed */
1327 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1328 mtlr r6
1329 stw r7, THREAD + RTAS_SP(r2)
1330 mtspr SPRN_SRR0,r8
1331 mtspr SPRN_SRR1,r9
1332 RFI
1333 1: tophys(r9,r1)
1334 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1335 lwz r9,8(r9) /* original msr value */
1336 addi r1,r1,INT_FRAME_SIZE
1337 li r0,0
1338 tophys(r7, r2)
1339 stw r0, THREAD + RTAS_SP(r7)
1340 mtspr SPRN_SRR0,r8
1341 mtspr SPRN_SRR1,r9
1342 RFI /* return to caller */
1343
1344 .globl machine_check_in_rtas
1345 machine_check_in_rtas:
1346 twi 31,0,0
1347 /* XXX load up BATs and panic */
1348
1349 #endif /* CONFIG_PPC_RTAS */
A mirror of Linus' kernel repository