[thirdparty/linux.git] / arch / arm64 / kernel / sdei.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2017 Arm Ltd.
#define pr_fmt(fmt) "sdei: " fmt

#include <linux/arm-smccc.h>
#include <linux/arm_sdei.h>
#include <linux/hardirq.h>
#include <linux/irqflags.h>
#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>

#include <asm/alternative.h>
#include <asm/kprobes.h>
#include <asm/mmu.h>
#include <asm/ptrace.h>
#include <asm/sections.h>
#include <asm/stacktrace.h>
#include <asm/sysreg.h>
#include <asm/vmap_stack.h>

unsigned long sdei_exit_mode;

/*
 * VMAP'd stacks checking for stack overflow on exception using sp as a scratch
 * register, meaning SDEI has to switch to its own stack. We need two stacks as
 * a critical event may interrupt a normal event that has just taken a
 * synchronous exception, and is using sp as scratch register. For a critical
 * event interrupting a normal event, we can't reliably tell if we were on the
 * sdei stack.
 * For now, we allocate stacks when the driver is probed.
 */
DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);

#ifdef CONFIG_VMAP_STACK
DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
#endif

static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
{
	unsigned long *p;

	p = per_cpu(*ptr, cpu);
	if (p) {
		per_cpu(*ptr, cpu) = NULL;
		vfree(p);
	}
}

static void free_sdei_stacks(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
		_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
	}
}

static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
{
	unsigned long *p;

	p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
	if (!p)
		return -ENOMEM;
	per_cpu(*ptr, cpu) = p;

	return 0;
}

static int init_sdei_stacks(void)
{
	int cpu;
	int err = 0;

	for_each_possible_cpu(cpu) {
		err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
		if (err)
			break;
		err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
		if (err)
			break;
	}

	if (err)
		free_sdei_stacks();

	return err;
}

static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
{
	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
	unsigned long high = low + SDEI_STACK_SIZE;

	if (!low)
		return false;

	if (sp < low || sp >= high)
		return false;

	if (info) {
		info->low = low;
		info->high = high;
		info->type = STACK_TYPE_SDEI_NORMAL;
	}

	return true;
}

static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
{
	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
	unsigned long high = low + SDEI_STACK_SIZE;

	if (!low)
		return false;

	if (sp < low || sp >= high)
		return false;

	if (info) {
		info->low = low;
		info->high = high;
		info->type = STACK_TYPE_SDEI_CRITICAL;
	}

	return true;
}

bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
{
	if (!IS_ENABLED(CONFIG_VMAP_STACK))
		return false;

	if (on_sdei_critical_stack(sp, info))
		return true;

	if (on_sdei_normal_stack(sp, info))
		return true;

	return false;
}

unsigned long sdei_arch_get_entry_point(int conduit)
{
	/*
	 * SDEI works between adjacent exception levels. If we booted at EL1 we
	 * assume a hypervisor is marshalling events. If we booted at EL2 and
	 * dropped to EL1 because we don't support VHE, then we can't support
	 * SDEI.
	 */
	if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
		pr_err("Not supported on this hardware/boot configuration\n");
		return 0;
	}

	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
		if (init_sdei_stacks())
			return 0;
	}

	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
	if (arm64_kernel_unmapped_at_el0()) {
		unsigned long offset;

		offset = (unsigned long)__sdei_asm_entry_trampoline -
			 (unsigned long)__entry_tramp_text_start;
		return TRAMP_VALIAS + offset;
	} else
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
		return (unsigned long)__sdei_asm_handler;

}

/*
 * __sdei_handler() returns one of:
 *  SDEI_EV_HANDLED -  success, return to the interrupted context.
 *  SDEI_EV_FAILED  -  failure, return this error code to firmare.
 *  virtual-address -  success, return to this address.
 */
static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
					     struct sdei_registered_event *arg)
{
	u32 mode;
	int i, err = 0;
	int clobbered_registers = 4;
	u64 elr = read_sysreg(elr_el1);
	u32 kernel_mode = read_sysreg(CurrentEL) | 1;	/* +SPSel */
	unsigned long vbar = read_sysreg(vbar_el1);

	if (arm64_kernel_unmapped_at_el0())
		clobbered_registers++;

	/* Retrieve the missing registers values */
	for (i = 0; i < clobbered_registers; i++) {
		/* from within the handler, this call always succeeds */
		sdei_api_event_context(i, &regs->regs[i]);
	}

	/*
	 * We didn't take an exception to get here, set PAN. UAO will be cleared
	 * by sdei_event_handler()s set_fs(USER_DS) call.
	 */
	__uaccess_enable_hw_pan();

	err = sdei_event_handler(regs, arg);
	if (err)
		return SDEI_EV_FAILED;

	if (elr != read_sysreg(elr_el1)) {
		/*
		 * We took a synchronous exception from the SDEI handler.
		 * This could deadlock, and if you interrupt KVM it will
		 * hyp-panic instead.
		 */
		pr_warn("unsafe: exception during handler\n");
	}

	mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);

	/*
	 * If we interrupted the kernel with interrupts masked, we always go
	 * back to wherever we came from.
	 */
	if (mode == kernel_mode && !interrupts_enabled(regs))
		return SDEI_EV_HANDLED;

	/*
	 * Otherwise, we pretend this was an IRQ. This lets user space tasks
	 * receive signals before we return to them, and KVM to invoke it's
	 * world switch to do the same.
	 *
	 * See DDI0487B.a Table D1-7 'Vector offsets from vector table base
	 * address'.
	 */
	if (mode == kernel_mode)
		return vbar + 0x280;
	else if (mode & PSR_MODE32_BIT)
		return vbar + 0x680;

	return vbar + 0x480;
}


asmlinkage __kprobes notrace unsigned long
__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
{
	unsigned long ret;

	nmi_enter();

	ret = _sdei_handler(regs, arg);

	nmi_exit();

	return ret;
}
Commit	Line	Data
f5df2696 JM	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (C) 2017 Arm Ltd.
	3	#define pr_fmt(fmt) "sdei: " fmt
	4
e6ea4651	5	#include <linux/arm-smccc.h>
f5df2696 JM	6	#include <linux/arm_sdei.h>
	7	#include <linux/hardirq.h>
	8	#include <linux/irqflags.h>
	9	#include <linux/sched/task_stack.h>
	10	#include <linux/uaccess.h>
	11
	12	#include <asm/alternative.h>
	13	#include <asm/kprobes.h>
79e9aa59	14	#include <asm/mmu.h>
f5df2696	15	#include <asm/ptrace.h>
79e9aa59	16	#include <asm/sections.h>
8a1ccfbc	17	#include <asm/stacktrace.h>
f5df2696 JM	18	#include <asm/sysreg.h>
	19	#include <asm/vmap_stack.h>
	20
	21	unsigned long sdei_exit_mode;
	22
	23	/*
	24	* VMAP'd stacks checking for stack overflow on exception using sp as a scratch
	25	* register, meaning SDEI has to switch to its own stack. We need two stacks as
	26	* a critical event may interrupt a normal event that has just taken a
	27	* synchronous exception, and is using sp as scratch register. For a critical
	28	* event interrupting a normal event, we can't reliably tell if we were on the
	29	* sdei stack.
	30	* For now, we allocate stacks when the driver is probed.
	31	*/
	32	DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
	33	DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
	34
	35	#ifdef CONFIG_VMAP_STACK
	36	DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
	37	DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
	38	#endif
	39
	40	static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
	41	{
	42	unsigned long *p;
	43
	44	p = per_cpu(*ptr, cpu);
	45	if (p) {
	46	per_cpu(*ptr, cpu) = NULL;
	47	vfree(p);
	48	}
	49	}
	50
	51	static void free_sdei_stacks(void)
	52	{
	53	int cpu;
	54
	55	for_each_possible_cpu(cpu) {
	56	_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
	57	_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
	58	}
	59	}
	60
	61	static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
	62	{
	63	unsigned long *p;
	64
	65	p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
	66	if (!p)
	67	return -ENOMEM;
	68	per_cpu(*ptr, cpu) = p;
	69
	70	return 0;
	71	}
	72
	73	static int init_sdei_stacks(void)
	74	{
	75	int cpu;
	76	int err = 0;
	77
	78	for_each_possible_cpu(cpu) {
	79	err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
	80	if (err)
	81	break;
82	err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
83	if (err)
84	break;
85	}
86
87	if (err)
88	free_sdei_stacks();
89
90	return err;
91	}
92
eab1cecc	93	static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
f5df2696	94	{
8a1ccfbc LA	95	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
8a1ccfbc LA	96	unsigned long high = low + SDEI_STACK_SIZE;
f5df2696	97
1c418608 WL	98	if (!low)
	99	return false;
	100
8a1ccfbc	101	if (sp < low \|\| sp >= high)
f5df2696 JM	102	return false;
f5df2696 JM	103
8a1ccfbc LA	104	if (info) {
	105	info->low = low;
	106	info->high = high;
	107	info->type = STACK_TYPE_SDEI_NORMAL;
	108	}
f5df2696	109
8a1ccfbc LA	110	return true;
	111	}
	112
eab1cecc	113	static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
8a1ccfbc LA	114	{
	115	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
	116	unsigned long high = low + SDEI_STACK_SIZE;
	117
1c418608 WL	118	if (!low)
	119	return false;
	120
8a1ccfbc LA	121	if (sp < low \|\| sp >= high)
	122	return false;
	123
	124	if (info) {
	125	info->low = low;
	126	info->high = high;
	127	info->type = STACK_TYPE_SDEI_CRITICAL;
	128	}
	129
	130	return true;
	131	}
	132
eab1cecc	133	bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
8a1ccfbc LA	134	{
	135	if (!IS_ENABLED(CONFIG_VMAP_STACK))
	136	return false;
	137
	138	if (on_sdei_critical_stack(sp, info))
f5df2696 JM	139	return true;
f5df2696 JM	140
8a1ccfbc LA	141	if (on_sdei_normal_stack(sp, info))
8a1ccfbc LA	142	return true;
f5df2696	143
8a1ccfbc	144	return false;
f5df2696 JM	145	}
	146
	147	unsigned long sdei_arch_get_entry_point(int conduit)
	148	{
	149	/*
	150	* SDEI works between adjacent exception levels. If we booted at EL1 we
	151	* assume a hypervisor is marshalling events. If we booted at EL2 and
	152	* dropped to EL1 because we don't support VHE, then we can't support
	153	* SDEI.
	154	*/
	155	if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
	156	pr_err("Not supported on this hardware/boot configuration\n");
	157	return 0;
	158	}
	159
	160	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
	161	if (init_sdei_stacks())
	162	return 0;
	163	}
	164
e6ea4651	165	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
79e9aa59 JM	166
	167	#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
	168	if (arm64_kernel_unmapped_at_el0()) {
	169	unsigned long offset;
	170
	171	offset = (unsigned long)__sdei_asm_entry_trampoline -
	172	(unsigned long)__entry_tramp_text_start;
	173	return TRAMP_VALIAS + offset;
	174	} else
	175	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
	176	return (unsigned long)__sdei_asm_handler;
	177
f5df2696 JM	178	}
	179
	180	/*
	181	* __sdei_handler() returns one of:
	182	* SDEI_EV_HANDLED - success, return to the interrupted context.
	183	* SDEI_EV_FAILED - failure, return this error code to firmare.
	184	* virtual-address - success, return to this address.
	185	*/
	186	static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
	187	struct sdei_registered_event *arg)
	188	{
	189	u32 mode;
	190	int i, err = 0;
79e9aa59	191	int clobbered_registers = 4;
f5df2696 JM	192	u64 elr = read_sysreg(elr_el1);
	193	u32 kernel_mode = read_sysreg(CurrentEL) \| 1; /* +SPSel */
	194	unsigned long vbar = read_sysreg(vbar_el1);
	195
79e9aa59 JM	196	if (arm64_kernel_unmapped_at_el0())
	197	clobbered_registers++;
	198
f5df2696 JM	199	/* Retrieve the missing registers values */
	200	for (i = 0; i < clobbered_registers; i++) {
	201	/* from within the handler, this call always succeeds */
	202	sdei_api_event_context(i, &regs->regs[i]);
	203	}
	204
	205	/*
	206	* We didn't take an exception to get here, set PAN. UAO will be cleared
	207	* by sdei_event_handler()s set_fs(USER_DS) call.
	208	*/
	209	__uaccess_enable_hw_pan();
	210
	211	err = sdei_event_handler(regs, arg);
	212	if (err)
	213	return SDEI_EV_FAILED;
	214
	215	if (elr != read_sysreg(elr_el1)) {
	216	/*
	217	* We took a synchronous exception from the SDEI handler.
	218	* This could deadlock, and if you interrupt KVM it will
	219	* hyp-panic instead.
	220	*/
	221	pr_warn("unsafe: exception during handler\n");
	222	}
	223
	224	mode = regs->pstate & (PSR_MODE32_BIT \| PSR_MODE_MASK);
	225
	226	/*
	227	* If we interrupted the kernel with interrupts masked, we always go
	228	* back to wherever we came from.
	229	*/
	230	if (mode == kernel_mode && !interrupts_enabled(regs))
	231	return SDEI_EV_HANDLED;
	232
	233	/*
	234	* Otherwise, we pretend this was an IRQ. This lets user space tasks
	235	* receive signals before we return to them, and KVM to invoke it's
	236	* world switch to do the same.
	237	*
	238	* See DDI0487B.a Table D1-7 'Vector offsets from vector table base
	239	* address'.
	240	*/
	241	if (mode == kernel_mode)
	242	return vbar + 0x280;
	243	else if (mode & PSR_MODE32_BIT)
	244	return vbar + 0x680;
	245
	246	return vbar + 0x480;
	247	}
	248
	249
	250	asmlinkage __kprobes notrace unsigned long
	251	__sdei_handler(struct pt_regs regs, struct sdei_registered_event arg)
	252	{
	253	unsigned long ret;
f5df2696	254
69ea03b5	255	nmi_enter();
f5df2696 JM	256
	257	ret = _sdei_handler(regs, arg);
	258
69ea03b5	259	nmi_exit();
f5df2696 JM	260
	261	return ret;
	262	}