]>
Commit | Line | Data |
---|---|---|
68b34588 NP |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | ||
3 | #include <linux/err.h> | |
4 | #include <asm/asm-prototypes.h> | |
5 | #include <asm/book3s/64/kup-radix.h> | |
6 | #include <asm/cputime.h> | |
7 | #include <asm/hw_irq.h> | |
8 | #include <asm/kprobes.h> | |
9 | #include <asm/paca.h> | |
10 | #include <asm/ptrace.h> | |
11 | #include <asm/reg.h> | |
12 | #include <asm/signal.h> | |
13 | #include <asm/switch_to.h> | |
14 | #include <asm/syscall.h> | |
15 | #include <asm/time.h> | |
16 | #include <asm/unistd.h> | |
17 | ||
18 | typedef long (*syscall_fn)(long, long, long, long, long, long); | |
19 | ||
5f0b6ac3 NP |
20 | /* Has to run notrace because it is entered not completely "reconciled" */ |
21 | notrace long system_call_exception(long r3, long r4, long r5, | |
22 | long r6, long r7, long r8, | |
23 | unsigned long r0, struct pt_regs *regs) | |
68b34588 | 24 | { |
68b34588 NP |
25 | syscall_fn f; |
26 | ||
5f0b6ac3 NP |
27 | if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) |
28 | BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); | |
29 | ||
30 | trace_hardirqs_off(); /* finish reconciling */ | |
31 | ||
6cc0c16d NP |
32 | if (IS_ENABLED(CONFIG_PPC_BOOK3S)) |
33 | BUG_ON(!(regs->msr & MSR_RI)); | |
68b34588 | 34 | BUG_ON(!(regs->msr & MSR_PR)); |
6cc0c16d NP |
35 | BUG_ON(!FULL_REGS(regs)); |
36 | BUG_ON(regs->softe != IRQS_ENABLED); | |
68b34588 | 37 | |
c0d7dcf8 NP |
38 | kuap_check_amr(); |
39 | ||
68b34588 NP |
40 | account_cpu_user_entry(); |
41 | ||
42 | #ifdef CONFIG_PPC_SPLPAR | |
43 | if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && | |
44 | firmware_has_feature(FW_FEATURE_SPLPAR)) { | |
45 | struct lppaca *lp = local_paca->lppaca_ptr; | |
46 | ||
47 | if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) | |
48 | accumulate_stolen_time(); | |
49 | } | |
50 | #endif | |
51 | ||
68b34588 NP |
52 | /* |
53 | * This is not required for the syscall exit path, but makes the | |
54 | * stack frame look nicer. If this was initialised in the first stack | |
55 | * frame, or if the unwinder was taught the first stack frame always | |
56 | * returns to user with IRQS_ENABLED, this store could be avoided! | |
57 | */ | |
58 | regs->softe = IRQS_ENABLED; | |
59 | ||
5f0b6ac3 | 60 | local_irq_enable(); |
68b34588 | 61 | |
0a7601b6 | 62 | if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) { |
68b34588 NP |
63 | /* |
64 | * We use the return value of do_syscall_trace_enter() as the | |
65 | * syscall number. If the syscall was rejected for any reason | |
66 | * do_syscall_trace_enter() returns an invalid syscall number | |
67 | * and the test against NR_syscalls will fail and the return | |
68 | * value to be used is in regs->gpr[3]. | |
69 | */ | |
70 | r0 = do_syscall_trace_enter(regs); | |
71 | if (unlikely(r0 >= NR_syscalls)) | |
72 | return regs->gpr[3]; | |
73 | r3 = regs->gpr[3]; | |
74 | r4 = regs->gpr[4]; | |
75 | r5 = regs->gpr[5]; | |
76 | r6 = regs->gpr[6]; | |
77 | r7 = regs->gpr[7]; | |
78 | r8 = regs->gpr[8]; | |
79 | ||
80 | } else if (unlikely(r0 >= NR_syscalls)) { | |
81 | return -ENOSYS; | |
82 | } | |
83 | ||
84 | /* May be faster to do array_index_nospec? */ | |
85 | barrier_nospec(); | |
86 | ||
0a7601b6 | 87 | if (unlikely(is_32bit_task())) { |
68b34588 NP |
88 | f = (void *)compat_sys_call_table[r0]; |
89 | ||
90 | r3 &= 0x00000000ffffffffULL; | |
91 | r4 &= 0x00000000ffffffffULL; | |
92 | r5 &= 0x00000000ffffffffULL; | |
93 | r6 &= 0x00000000ffffffffULL; | |
94 | r7 &= 0x00000000ffffffffULL; | |
95 | r8 &= 0x00000000ffffffffULL; | |
96 | ||
97 | } else { | |
98 | f = (void *)sys_call_table[r0]; | |
99 | } | |
100 | ||
101 | return f(r3, r4, r5, r6, r7, r8); | |
102 | } | |
103 | ||
104 | /* | |
105 | * This should be called after a syscall returns, with r3 the return value | |
106 | * from the syscall. If this function returns non-zero, the system call | |
107 | * exit assembly should additionally load all GPR registers and CTR and XER | |
108 | * from the interrupt frame. | |
109 | * | |
110 | * The function graph tracer can not trace the return side of this function, | |
111 | * because RI=0 and soft mask state is "unreconciled", so it is marked notrace. | |
112 | */ | |
113 | notrace unsigned long syscall_exit_prepare(unsigned long r3, | |
114 | struct pt_regs *regs) | |
115 | { | |
116 | unsigned long *ti_flagsp = ¤t_thread_info()->flags; | |
117 | unsigned long ti_flags; | |
118 | unsigned long ret = 0; | |
119 | ||
c0d7dcf8 NP |
120 | kuap_check_amr(); |
121 | ||
68b34588 NP |
122 | regs->result = r3; |
123 | ||
124 | /* Check whether the syscall is issued inside a restartable sequence */ | |
125 | rseq_syscall(regs); | |
126 | ||
127 | ti_flags = *ti_flagsp; | |
128 | ||
129 | if (unlikely(r3 >= (unsigned long)-MAX_ERRNO)) { | |
130 | if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { | |
131 | r3 = -r3; | |
132 | regs->ccr |= 0x10000000; /* Set SO bit in CR */ | |
133 | } | |
134 | } | |
135 | ||
136 | if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) { | |
137 | if (ti_flags & _TIF_RESTOREALL) | |
138 | ret = _TIF_RESTOREALL; | |
139 | else | |
140 | regs->gpr[3] = r3; | |
141 | clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp); | |
142 | } else { | |
143 | regs->gpr[3] = r3; | |
144 | } | |
145 | ||
146 | if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) { | |
147 | do_syscall_trace_leave(regs); | |
148 | ret |= _TIF_RESTOREALL; | |
149 | } | |
150 | ||
151 | again: | |
152 | local_irq_disable(); | |
153 | ti_flags = READ_ONCE(*ti_flagsp); | |
154 | while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { | |
155 | local_irq_enable(); | |
156 | if (ti_flags & _TIF_NEED_RESCHED) { | |
157 | schedule(); | |
158 | } else { | |
159 | /* | |
160 | * SIGPENDING must restore signal handler function | |
161 | * argument GPRs, and some non-volatiles (e.g., r1). | |
162 | * Restore all for now. This could be made lighter. | |
163 | */ | |
164 | if (ti_flags & _TIF_SIGPENDING) | |
165 | ret |= _TIF_RESTOREALL; | |
166 | do_notify_resume(regs, ti_flags); | |
167 | } | |
168 | local_irq_disable(); | |
169 | ti_flags = READ_ONCE(*ti_flagsp); | |
170 | } | |
171 | ||
172 | if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { | |
173 | if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && | |
174 | unlikely((ti_flags & _TIF_RESTORE_TM))) { | |
175 | restore_tm_state(regs); | |
176 | } else { | |
177 | unsigned long mathflags = MSR_FP; | |
178 | ||
179 | if (cpu_has_feature(CPU_FTR_VSX)) | |
180 | mathflags |= MSR_VEC | MSR_VSX; | |
181 | else if (cpu_has_feature(CPU_FTR_ALTIVEC)) | |
182 | mathflags |= MSR_VEC; | |
183 | ||
184 | if ((regs->msr & mathflags) != mathflags) | |
185 | restore_math(regs); | |
186 | } | |
187 | } | |
188 | ||
189 | /* This must be done with RI=1 because tracing may touch vmaps */ | |
190 | trace_hardirqs_on(); | |
191 | ||
192 | /* This pattern matches prep_irq_for_idle */ | |
193 | __hard_EE_RI_disable(); | |
0094368e | 194 | if (unlikely(lazy_irq_pending_nocheck())) { |
68b34588 NP |
195 | __hard_RI_enable(); |
196 | trace_hardirqs_off(); | |
197 | local_paca->irq_happened |= PACA_IRQ_HARD_DIS; | |
198 | local_irq_enable(); | |
6cc0c16d | 199 | /* Took an interrupt, may have more exit work to do. */ |
68b34588 NP |
200 | goto again; |
201 | } | |
202 | local_paca->irq_happened = 0; | |
203 | irq_soft_mask_set(IRQS_ENABLED); | |
204 | ||
205 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM | |
206 | local_paca->tm_scratch = regs->msr; | |
207 | #endif | |
208 | ||
68b34588 NP |
209 | account_cpu_user_exit(); |
210 | ||
211 | return ret; | |
212 | } | |
6cc0c16d NP |
213 | |
214 | #ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */ | |
215 | notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr) | |
216 | { | |
217 | #ifdef CONFIG_PPC_BOOK3E | |
218 | struct thread_struct *ts = ¤t->thread; | |
219 | #endif | |
220 | unsigned long *ti_flagsp = ¤t_thread_info()->flags; | |
221 | unsigned long ti_flags; | |
222 | unsigned long flags; | |
223 | unsigned long ret = 0; | |
224 | ||
225 | if (IS_ENABLED(CONFIG_PPC_BOOK3S)) | |
226 | BUG_ON(!(regs->msr & MSR_RI)); | |
227 | BUG_ON(!(regs->msr & MSR_PR)); | |
228 | BUG_ON(!FULL_REGS(regs)); | |
229 | BUG_ON(regs->softe != IRQS_ENABLED); | |
230 | ||
c0d7dcf8 NP |
231 | kuap_check_amr(); |
232 | ||
6cc0c16d NP |
233 | local_irq_save(flags); |
234 | ||
235 | again: | |
236 | ti_flags = READ_ONCE(*ti_flagsp); | |
237 | while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { | |
238 | local_irq_enable(); /* returning to user: may enable */ | |
239 | if (ti_flags & _TIF_NEED_RESCHED) { | |
240 | schedule(); | |
241 | } else { | |
242 | if (ti_flags & _TIF_SIGPENDING) | |
243 | ret |= _TIF_RESTOREALL; | |
244 | do_notify_resume(regs, ti_flags); | |
245 | } | |
246 | local_irq_disable(); | |
247 | ti_flags = READ_ONCE(*ti_flagsp); | |
248 | } | |
249 | ||
250 | if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { | |
251 | if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && | |
252 | unlikely((ti_flags & _TIF_RESTORE_TM))) { | |
253 | restore_tm_state(regs); | |
254 | } else { | |
255 | unsigned long mathflags = MSR_FP; | |
256 | ||
257 | if (cpu_has_feature(CPU_FTR_VSX)) | |
258 | mathflags |= MSR_VEC | MSR_VSX; | |
259 | else if (cpu_has_feature(CPU_FTR_ALTIVEC)) | |
260 | mathflags |= MSR_VEC; | |
261 | ||
262 | if ((regs->msr & mathflags) != mathflags) | |
263 | restore_math(regs); | |
264 | } | |
265 | } | |
266 | ||
267 | trace_hardirqs_on(); | |
268 | __hard_EE_RI_disable(); | |
0094368e | 269 | if (unlikely(lazy_irq_pending_nocheck())) { |
6cc0c16d NP |
270 | __hard_RI_enable(); |
271 | trace_hardirqs_off(); | |
272 | local_paca->irq_happened |= PACA_IRQ_HARD_DIS; | |
273 | local_irq_enable(); | |
274 | local_irq_disable(); | |
275 | /* Took an interrupt, may have more exit work to do. */ | |
276 | goto again; | |
277 | } | |
278 | local_paca->irq_happened = 0; | |
279 | irq_soft_mask_set(IRQS_ENABLED); | |
280 | ||
281 | #ifdef CONFIG_PPC_BOOK3E | |
282 | if (unlikely(ts->debug.dbcr0 & DBCR0_IDM)) { | |
283 | /* | |
284 | * Check to see if the dbcr0 register is set up to debug. | |
285 | * Use the internal debug mode bit to do this. | |
286 | */ | |
287 | mtmsr(mfmsr() & ~MSR_DE); | |
288 | mtspr(SPRN_DBCR0, ts->debug.dbcr0); | |
289 | mtspr(SPRN_DBSR, -1); | |
290 | } | |
291 | #endif | |
292 | ||
293 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM | |
294 | local_paca->tm_scratch = regs->msr; | |
295 | #endif | |
296 | ||
6cc0c16d NP |
297 | account_cpu_user_exit(); |
298 | ||
299 | return ret; | |
300 | } | |
301 | ||
302 | void unrecoverable_exception(struct pt_regs *regs); | |
303 | void preempt_schedule_irq(void); | |
304 | ||
305 | notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr) | |
306 | { | |
307 | unsigned long *ti_flagsp = ¤t_thread_info()->flags; | |
308 | unsigned long flags; | |
309 | unsigned long ret = 0; | |
310 | ||
311 | if (IS_ENABLED(CONFIG_PPC_BOOK3S) && unlikely(!(regs->msr & MSR_RI))) | |
312 | unrecoverable_exception(regs); | |
313 | BUG_ON(regs->msr & MSR_PR); | |
314 | BUG_ON(!FULL_REGS(regs)); | |
315 | ||
c0d7dcf8 NP |
316 | kuap_check_amr(); |
317 | ||
6cc0c16d NP |
318 | if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) { |
319 | clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp); | |
320 | ret = 1; | |
321 | } | |
322 | ||
323 | local_irq_save(flags); | |
324 | ||
325 | if (regs->softe == IRQS_ENABLED) { | |
326 | /* Returning to a kernel context with local irqs enabled. */ | |
327 | WARN_ON_ONCE(!(regs->msr & MSR_EE)); | |
328 | again: | |
329 | if (IS_ENABLED(CONFIG_PREEMPT)) { | |
330 | /* Return to preemptible kernel context */ | |
331 | if (unlikely(*ti_flagsp & _TIF_NEED_RESCHED)) { | |
332 | if (preempt_count() == 0) | |
333 | preempt_schedule_irq(); | |
334 | } | |
335 | } | |
336 | ||
337 | trace_hardirqs_on(); | |
338 | __hard_EE_RI_disable(); | |
0094368e | 339 | if (unlikely(lazy_irq_pending_nocheck())) { |
6cc0c16d NP |
340 | __hard_RI_enable(); |
341 | irq_soft_mask_set(IRQS_ALL_DISABLED); | |
342 | trace_hardirqs_off(); | |
343 | local_paca->irq_happened |= PACA_IRQ_HARD_DIS; | |
344 | /* | |
345 | * Can't local_irq_restore to replay if we were in | |
346 | * interrupt context. Must replay directly. | |
347 | */ | |
348 | if (irqs_disabled_flags(flags)) { | |
349 | replay_soft_interrupts(); | |
350 | } else { | |
351 | local_irq_restore(flags); | |
352 | local_irq_save(flags); | |
353 | } | |
354 | /* Took an interrupt, may have more exit work to do. */ | |
355 | goto again; | |
356 | } | |
357 | local_paca->irq_happened = 0; | |
358 | irq_soft_mask_set(IRQS_ENABLED); | |
359 | } else { | |
360 | /* Returning to a kernel context with local irqs disabled. */ | |
361 | __hard_EE_RI_disable(); | |
362 | if (regs->msr & MSR_EE) | |
363 | local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; | |
364 | } | |
365 | ||
366 | ||
367 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM | |
368 | local_paca->tm_scratch = regs->msr; | |
369 | #endif | |
370 | ||
371 | /* | |
372 | * We don't need to restore AMR on the way back to userspace for KUAP. | |
373 | * The value of AMR only matters while we're in the kernel. | |
374 | */ | |
375 | kuap_restore_amr(regs); | |
376 | ||
377 | return ret; | |
378 | } | |
379 | #endif |