]>
git.ipfire.org Git - thirdparty/linux.git/blob - arch/arm64/kernel/entry-common.c
1 // SPDX-License-Identifier: GPL-2.0
3 * Exception handling code
5 * Copyright (C) 2019 ARM Ltd.
8 #include <linux/context_tracking.h>
9 #include <linux/ptrace.h>
10 #include <linux/thread_info.h>
12 #include <asm/cpufeature.h>
13 #include <asm/daifflags.h>
15 #include <asm/exception.h>
16 #include <asm/kprobes.h>
18 #include <asm/sysreg.h>
20 static void notrace
el1_abort(struct pt_regs
*regs
, unsigned long esr
)
22 unsigned long far
= read_sysreg(far_el1
);
24 local_daif_inherit(regs
);
25 far
= untagged_addr(far
);
26 do_mem_abort(far
, esr
, regs
);
28 NOKPROBE_SYMBOL(el1_abort
);
30 static void notrace
el1_pc(struct pt_regs
*regs
, unsigned long esr
)
32 unsigned long far
= read_sysreg(far_el1
);
34 local_daif_inherit(regs
);
35 do_sp_pc_abort(far
, esr
, regs
);
37 NOKPROBE_SYMBOL(el1_pc
);
39 static void notrace
el1_undef(struct pt_regs
*regs
)
41 local_daif_inherit(regs
);
44 NOKPROBE_SYMBOL(el1_undef
);
46 static void notrace
el1_inv(struct pt_regs
*regs
, unsigned long esr
)
48 local_daif_inherit(regs
);
49 bad_mode(regs
, 0, esr
);
51 NOKPROBE_SYMBOL(el1_inv
);
53 static void notrace
el1_dbg(struct pt_regs
*regs
, unsigned long esr
)
55 unsigned long far
= read_sysreg(far_el1
);
58 * The CPU masked interrupts, and we are leaving them masked during
59 * do_debug_exception(). Update PMR as if we had called
62 if (system_uses_irq_prio_masking())
63 gic_write_pmr(GIC_PRIO_IRQON
| GIC_PRIO_PSR_I_SET
);
65 do_debug_exception(far
, esr
, regs
);
67 NOKPROBE_SYMBOL(el1_dbg
);
69 asmlinkage
void notrace
el1_sync_handler(struct pt_regs
*regs
)
71 unsigned long esr
= read_sysreg(esr_el1
);
73 switch (ESR_ELx_EC(esr
)) {
74 case ESR_ELx_EC_DABT_CUR
:
75 case ESR_ELx_EC_IABT_CUR
:
79 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
80 * recursive exception when trying to push the initial pt_regs.
82 case ESR_ELx_EC_PC_ALIGN
:
85 case ESR_ELx_EC_SYS64
:
86 case ESR_ELx_EC_UNKNOWN
:
89 case ESR_ELx_EC_BREAKPT_CUR
:
90 case ESR_ELx_EC_SOFTSTP_CUR
:
91 case ESR_ELx_EC_WATCHPT_CUR
:
92 case ESR_ELx_EC_BRK64
:
99 NOKPROBE_SYMBOL(el1_sync_handler
);
101 static void notrace
el0_da(struct pt_regs
*regs
, unsigned long esr
)
103 unsigned long far
= read_sysreg(far_el1
);
106 local_daif_restore(DAIF_PROCCTX
);
107 far
= untagged_addr(far
);
108 do_mem_abort(far
, esr
, regs
);
110 NOKPROBE_SYMBOL(el0_da
);
112 static void notrace
el0_ia(struct pt_regs
*regs
, unsigned long esr
)
114 unsigned long far
= read_sysreg(far_el1
);
117 * We've taken an instruction abort from userspace and not yet
118 * re-enabled IRQs. If the address is a kernel address, apply
119 * BP hardening prior to enabling IRQs and pre-emption.
121 if (!is_ttbr0_addr(far
))
122 arm64_apply_bp_hardening();
125 local_daif_restore(DAIF_PROCCTX
);
126 do_mem_abort(far
, esr
, regs
);
128 NOKPROBE_SYMBOL(el0_ia
);
130 static void notrace
el0_fpsimd_acc(struct pt_regs
*regs
, unsigned long esr
)
133 local_daif_restore(DAIF_PROCCTX
);
134 do_fpsimd_acc(esr
, regs
);
136 NOKPROBE_SYMBOL(el0_fpsimd_acc
);
138 static void notrace
el0_sve_acc(struct pt_regs
*regs
, unsigned long esr
)
141 local_daif_restore(DAIF_PROCCTX
);
142 do_sve_acc(esr
, regs
);
144 NOKPROBE_SYMBOL(el0_sve_acc
);
146 static void notrace
el0_fpsimd_exc(struct pt_regs
*regs
, unsigned long esr
)
149 local_daif_restore(DAIF_PROCCTX
);
150 do_fpsimd_exc(esr
, regs
);
152 NOKPROBE_SYMBOL(el0_fpsimd_exc
);
154 static void notrace
el0_sys(struct pt_regs
*regs
, unsigned long esr
)
157 local_daif_restore(DAIF_PROCCTX
);
158 do_sysinstr(esr
, regs
);
160 NOKPROBE_SYMBOL(el0_sys
);
162 static void notrace
el0_pc(struct pt_regs
*regs
, unsigned long esr
)
164 unsigned long far
= read_sysreg(far_el1
);
166 if (!is_ttbr0_addr(instruction_pointer(regs
)))
167 arm64_apply_bp_hardening();
170 local_daif_restore(DAIF_PROCCTX
);
171 do_sp_pc_abort(far
, esr
, regs
);
173 NOKPROBE_SYMBOL(el0_pc
);
175 static void notrace
el0_sp(struct pt_regs
*regs
, unsigned long esr
)
178 local_daif_restore(DAIF_PROCCTX
);
179 do_sp_pc_abort(regs
->sp
, esr
, regs
);
181 NOKPROBE_SYMBOL(el0_sp
);
183 static void notrace
el0_undef(struct pt_regs
*regs
)
186 local_daif_restore(DAIF_PROCCTX
);
189 NOKPROBE_SYMBOL(el0_undef
);
191 static void notrace
el0_bti(struct pt_regs
*regs
)
194 local_daif_restore(DAIF_PROCCTX
);
197 NOKPROBE_SYMBOL(el0_bti
);
199 static void notrace
el0_inv(struct pt_regs
*regs
, unsigned long esr
)
202 local_daif_restore(DAIF_PROCCTX
);
203 bad_el0_sync(regs
, 0, esr
);
205 NOKPROBE_SYMBOL(el0_inv
);
207 static void notrace
el0_dbg(struct pt_regs
*regs
, unsigned long esr
)
209 /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
210 unsigned long far
= read_sysreg(far_el1
);
212 if (system_uses_irq_prio_masking())
213 gic_write_pmr(GIC_PRIO_IRQON
| GIC_PRIO_PSR_I_SET
);
216 do_debug_exception(far
, esr
, regs
);
217 local_daif_restore(DAIF_PROCCTX_NOIRQ
);
219 NOKPROBE_SYMBOL(el0_dbg
);
221 static void notrace
el0_svc(struct pt_regs
*regs
)
223 if (system_uses_irq_prio_masking())
224 gic_write_pmr(GIC_PRIO_IRQON
| GIC_PRIO_PSR_I_SET
);
228 NOKPROBE_SYMBOL(el0_svc
);
230 asmlinkage
void notrace
el0_sync_handler(struct pt_regs
*regs
)
232 unsigned long esr
= read_sysreg(esr_el1
);
234 switch (ESR_ELx_EC(esr
)) {
235 case ESR_ELx_EC_SVC64
:
238 case ESR_ELx_EC_DABT_LOW
:
241 case ESR_ELx_EC_IABT_LOW
:
244 case ESR_ELx_EC_FP_ASIMD
:
245 el0_fpsimd_acc(regs
, esr
);
248 el0_sve_acc(regs
, esr
);
250 case ESR_ELx_EC_FP_EXC64
:
251 el0_fpsimd_exc(regs
, esr
);
253 case ESR_ELx_EC_SYS64
:
257 case ESR_ELx_EC_SP_ALIGN
:
260 case ESR_ELx_EC_PC_ALIGN
:
263 case ESR_ELx_EC_UNKNOWN
:
269 case ESR_ELx_EC_BREAKPT_LOW
:
270 case ESR_ELx_EC_SOFTSTP_LOW
:
271 case ESR_ELx_EC_WATCHPT_LOW
:
272 case ESR_ELx_EC_BRK64
:
279 NOKPROBE_SYMBOL(el0_sync_handler
);
282 static void notrace
el0_cp15(struct pt_regs
*regs
, unsigned long esr
)
285 local_daif_restore(DAIF_PROCCTX
);
286 do_cp15instr(esr
, regs
);
288 NOKPROBE_SYMBOL(el0_cp15
);
290 static void notrace
el0_svc_compat(struct pt_regs
*regs
)
292 if (system_uses_irq_prio_masking())
293 gic_write_pmr(GIC_PRIO_IRQON
| GIC_PRIO_PSR_I_SET
);
295 do_el0_svc_compat(regs
);
297 NOKPROBE_SYMBOL(el0_svc_compat
);
299 asmlinkage
void notrace
el0_sync_compat_handler(struct pt_regs
*regs
)
301 unsigned long esr
= read_sysreg(esr_el1
);
303 switch (ESR_ELx_EC(esr
)) {
304 case ESR_ELx_EC_SVC32
:
305 el0_svc_compat(regs
);
307 case ESR_ELx_EC_DABT_LOW
:
310 case ESR_ELx_EC_IABT_LOW
:
313 case ESR_ELx_EC_FP_ASIMD
:
314 el0_fpsimd_acc(regs
, esr
);
316 case ESR_ELx_EC_FP_EXC32
:
317 el0_fpsimd_exc(regs
, esr
);
319 case ESR_ELx_EC_PC_ALIGN
:
322 case ESR_ELx_EC_UNKNOWN
:
323 case ESR_ELx_EC_CP14_MR
:
324 case ESR_ELx_EC_CP14_LS
:
325 case ESR_ELx_EC_CP14_64
:
328 case ESR_ELx_EC_CP15_32
:
329 case ESR_ELx_EC_CP15_64
:
332 case ESR_ELx_EC_BREAKPT_LOW
:
333 case ESR_ELx_EC_SOFTSTP_LOW
:
334 case ESR_ELx_EC_WATCHPT_LOW
:
335 case ESR_ELx_EC_BKPT32
:
342 NOKPROBE_SYMBOL(el0_sync_compat_handler
);
343 #endif /* CONFIG_COMPAT */