[thirdparty/linux.git] / arch / powerpc / include / asm / book3s / 32 / kup.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_32_KUP_H
#define _ASM_POWERPC_BOOK3S_32_KUP_H

#include <asm/book3s/32/mmu-hash.h>

#ifdef __ASSEMBLY__

.macro kuep_update_sr	gpr1, gpr2		/* NEVER use r0 as gpr2 due to addis */
101:	mtsrin	\gpr1, \gpr2
	addi	\gpr1, \gpr1, 0x111		/* next VSID */
	rlwinm	\gpr1, \gpr1, 0, 0xf0ffffff	/* clear VSID overflow */
	addis	\gpr2, \gpr2, 0x1000		/* address of next segment */
	bdnz	101b
	isync
.endm

.macro kuep_lock	gpr1, gpr2
#ifdef CONFIG_PPC_KUEP
	li	\gpr1, NUM_USER_SEGMENTS
	li	\gpr2, 0
	mtctr	\gpr1
	mfsrin	\gpr1, \gpr2
	oris	\gpr1, \gpr1, SR_NX@h		/* set Nx */
	kuep_update_sr \gpr1, \gpr2
#endif
.endm

.macro kuep_unlock	gpr1, gpr2
#ifdef CONFIG_PPC_KUEP
	li	\gpr1, NUM_USER_SEGMENTS
	li	\gpr2, 0
	mtctr	\gpr1
	mfsrin	\gpr1, \gpr2
	rlwinm	\gpr1, \gpr1, 0, ~SR_NX		/* Clear Nx */
	kuep_update_sr \gpr1, \gpr2
#endif
.endm

#ifdef CONFIG_PPC_KUAP

.macro kuap_update_sr	gpr1, gpr2, gpr3	/* NEVER use r0 as gpr2 due to addis */
101:	mtsrin	\gpr1, \gpr2
	addi	\gpr1, \gpr1, 0x111		/* next VSID */
	rlwinm	\gpr1, \gpr1, 0, 0xf0ffffff	/* clear VSID overflow */
	addis	\gpr2, \gpr2, 0x1000		/* address of next segment */
	cmplw	\gpr2, \gpr3
	blt-	101b
	isync
.endm

.macro kuap_save_and_lock	sp, thread, gpr1, gpr2, gpr3
	lwz	\gpr2, KUAP(\thread)
	rlwinm.	\gpr3, \gpr2, 28, 0xf0000000
	stw	\gpr2, STACK_REGS_KUAP(\sp)
	beq+	102f
	li	\gpr1, 0
	stw	\gpr1, KUAP(\thread)
	mfsrin	\gpr1, \gpr2
	oris	\gpr1, \gpr1, SR_KS@h	/* set Ks */
	kuap_update_sr	\gpr1, \gpr2, \gpr3
102:
.endm

.macro kuap_restore	sp, current, gpr1, gpr2, gpr3
	lwz	\gpr2, STACK_REGS_KUAP(\sp)
	rlwinm.	\gpr3, \gpr2, 28, 0xf0000000
	stw	\gpr2, THREAD + KUAP(\current)
	beq+	102f
	mfsrin	\gpr1, \gpr2
	rlwinm	\gpr1, \gpr1, 0, ~SR_KS	/* Clear Ks */
	kuap_update_sr	\gpr1, \gpr2, \gpr3
102:
.endm

.macro kuap_check	current, gpr
#ifdef CONFIG_PPC_KUAP_DEBUG
	lwz	\gpr, KUAP(thread)
999:	twnei	\gpr, 0
	EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
#endif
.endm

#endif /* CONFIG_PPC_KUAP */

#else /* !__ASSEMBLY__ */

#ifdef CONFIG_PPC_KUAP

#include <linux/sched.h>

static inline void kuap_update_sr(u32 sr, u32 addr, u32 end)
{
	addr &= 0xf0000000;	/* align addr to start of segment */
	barrier();	/* make sure thread.kuap is updated before playing with SRs */
	while (addr < end) {
		mtsrin(sr, addr);
		sr += 0x111;		/* next VSID */
		sr &= 0xf0ffffff;	/* clear VSID overflow */
		addr += 0x10000000;	/* address of next segment */
	}
	isync();	/* Context sync required after mtsrin() */
}

static __always_inline void allow_user_access(void __user *to, const void __user *from,
					      u32 size, unsigned long dir)
{
	u32 addr, end;

	BUILD_BUG_ON(!__builtin_constant_p(dir));
	BUILD_BUG_ON(dir == KUAP_CURRENT);

	if (!(dir & KUAP_WRITE))
		return;

	addr = (__force u32)to;

	if (unlikely(addr >= TASK_SIZE || !size))
		return;

	end = min(addr + size, TASK_SIZE);

	current->thread.kuap = (addr & 0xf0000000) | ((((end - 1) >> 28) + 1) & 0xf);
	kuap_update_sr(mfsrin(addr) & ~SR_KS, addr, end);	/* Clear Ks */
}

static __always_inline void prevent_user_access(void __user *to, const void __user *from,
						u32 size, unsigned long dir)
{
	u32 addr, end;

	BUILD_BUG_ON(!__builtin_constant_p(dir));

	if (dir == KUAP_CURRENT) {
		u32 kuap = current->thread.kuap;

		if (unlikely(!kuap))
			return;

		addr = kuap & 0xf0000000;
		end = kuap << 28;
	} else if (dir & KUAP_WRITE) {
		addr = (__force u32)to;
		end = min(addr + size, TASK_SIZE);

		if (unlikely(addr >= TASK_SIZE || !size))
			return;
	} else {
		return;
	}

	current->thread.kuap = 0;
	kuap_update_sr(mfsrin(addr) | SR_KS, addr, end);	/* set Ks */
}

static inline unsigned long prevent_user_access_return(void)
{
	unsigned long flags = current->thread.kuap;
	unsigned long addr = flags & 0xf0000000;
	unsigned long end = flags << 28;
	void __user *to = (__force void __user *)addr;

	if (flags)
		prevent_user_access(to, to, end - addr, KUAP_READ_WRITE);

	return flags;
}

static inline void restore_user_access(unsigned long flags)
{
	unsigned long addr = flags & 0xf0000000;
	unsigned long end = flags << 28;
	void __user *to = (__force void __user *)addr;

	if (flags)
		allow_user_access(to, to, end - addr, KUAP_READ_WRITE);
}

static inline bool
bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
	unsigned long begin = regs->kuap & 0xf0000000;
	unsigned long end = regs->kuap << 28;

	if (!is_write)
		return false;

	return WARN(address < begin || address >= end,
		    "Bug: write fault blocked by segment registers !");
}

#endif /* CONFIG_PPC_KUAP */

#endif /* __ASSEMBLY__ */

#endif /* _ASM_POWERPC_BOOK3S_32_KUP_H */
Commit	Line	Data
31ed2b13 CL	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _ASM_POWERPC_BOOK3S_32_KUP_H
	3	#define _ASM_POWERPC_BOOK3S_32_KUP_H
	4
	5	#include <asm/book3s/32/mmu-hash.h>
	6
	7	#ifdef __ASSEMBLY__
	8
	9	.macro kuep_update_sr gpr1, gpr2 /* NEVER use r0 as gpr2 due to addis */
	10	101: mtsrin \gpr1, \gpr2
	11	addi \gpr1, \gpr1, 0x111 /* next VSID */
	12	rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
	13	addis \gpr2, \gpr2, 0x1000 /* address of next segment */
	14	bdnz 101b
	15	isync
	16	.endm
	17
	18	.macro kuep_lock gpr1, gpr2
	19	#ifdef CONFIG_PPC_KUEP
	20	li \gpr1, NUM_USER_SEGMENTS
	21	li \gpr2, 0
	22	mtctr \gpr1
	23	mfsrin \gpr1, \gpr2
	24	oris \gpr1, \gpr1, SR_NX@h /* set Nx */
	25	kuep_update_sr \gpr1, \gpr2
	26	#endif
	27	.endm
	28
	29	.macro kuep_unlock gpr1, gpr2
	30	#ifdef CONFIG_PPC_KUEP
	31	li \gpr1, NUM_USER_SEGMENTS
	32	li \gpr2, 0
	33	mtctr \gpr1
	34	mfsrin \gpr1, \gpr2
	35	rlwinm \gpr1, \gpr1, 0, ~SR_NX /* Clear Nx */
	36	kuep_update_sr \gpr1, \gpr2
	37	#endif
	38	.endm
	39
a68c31fc CL	40	#ifdef CONFIG_PPC_KUAP
	41
	42	.macro kuap_update_sr gpr1, gpr2, gpr3 /* NEVER use r0 as gpr2 due to addis */
	43	101: mtsrin \gpr1, \gpr2
	44	addi \gpr1, \gpr1, 0x111 /* next VSID */
	45	rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
	46	addis \gpr2, \gpr2, 0x1000 /* address of next segment */
	47	cmplw \gpr2, \gpr3
	48	blt- 101b
	49	isync
	50	.endm
	51
	52	.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
	53	lwz \gpr2, KUAP(\thread)
	54	rlwinm. \gpr3, \gpr2, 28, 0xf0000000
	55	stw \gpr2, STACK_REGS_KUAP(\sp)
	56	beq+ 102f
	57	li \gpr1, 0
	58	stw \gpr1, KUAP(\thread)
	59	mfsrin \gpr1, \gpr2
	60	oris \gpr1, \gpr1, SR_KS@h /* set Ks */
	61	kuap_update_sr \gpr1, \gpr2, \gpr3
	62	102:
	63	.endm
	64
	65	.macro kuap_restore sp, current, gpr1, gpr2, gpr3
	66	lwz \gpr2, STACK_REGS_KUAP(\sp)
	67	rlwinm. \gpr3, \gpr2, 28, 0xf0000000
	68	stw \gpr2, THREAD + KUAP(\current)
	69	beq+ 102f
	70	mfsrin \gpr1, \gpr2
	71	rlwinm \gpr1, \gpr1, 0, ~SR_KS /* Clear Ks */
	72	kuap_update_sr \gpr1, \gpr2, \gpr3
	73	102:
	74	.endm
	75
	76	.macro kuap_check current, gpr
	77	#ifdef CONFIG_PPC_KUAP_DEBUG
4833ce06	78	lwz \gpr, KUAP(thread)
a68c31fc CL	79	999: twnei \gpr, 0
	80	EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING \| BUGFLAG_ONCE)
	81	#endif
	82	.endm
	83
	84	#endif /* CONFIG_PPC_KUAP */
	85
	86	#else /* !__ASSEMBLY__ */
	87
	88	#ifdef CONFIG_PPC_KUAP
	89
	90	#include <linux/sched.h>
	91
	92	static inline void kuap_update_sr(u32 sr, u32 addr, u32 end)
	93	{
d10f60ae	94	addr &= 0xf0000000; /* align addr to start of segment */
a68c31fc CL	95	barrier(); /* make sure thread.kuap is updated before playing with SRs */
	96	while (addr < end) {
	97	mtsrin(sr, addr);
	98	sr += 0x111; /* next VSID */
	99	sr &= 0xf0ffffff; /* clear VSID overflow */
	100	addr += 0x10000000; /* address of next segment */
	101	}
	102	isync(); /* Context sync required after mtsrin() */
	103	}
	104
1d8f739b CL	105	static __always_inline void allow_user_access(void __user to, const void __user from,
1d8f739b CL	106	u32 size, unsigned long dir)
a68c31fc CL	107	{
	108	u32 addr, end;
	109
1d8f739b	110	BUILD_BUG_ON(!__builtin_constant_p(dir));
bedb4dbe CL	111	BUILD_BUG_ON(dir == KUAP_CURRENT);
bedb4dbe CL	112
1d8f739b	113	if (!(dir & KUAP_WRITE))
a68c31fc CL	114	return;
	115
	116	addr = (__force u32)to;
	117
88f8c080	118	if (unlikely(addr >= TASK_SIZE \|\| !size))
a68c31fc CL	119	return;
	120
	121	end = min(addr + size, TASK_SIZE);
bedb4dbe	122
a68c31fc CL	123	current->thread.kuap = (addr & 0xf0000000) \| ((((end - 1) >> 28) + 1) & 0xf);
	124	kuap_update_sr(mfsrin(addr) & ~SR_KS, addr, end); /* Clear Ks */
	125	}
	126
1d8f739b CL	127	static __always_inline void prevent_user_access(void __user to, const void __user from,
1d8f739b CL	128	u32 size, unsigned long dir)
a68c31fc	129	{
88f8c080	130	u32 addr, end;
a68c31fc	131
1d8f739b	132	BUILD_BUG_ON(!__builtin_constant_p(dir));
1d8f739b	133
bedb4dbe CL	134	if (dir == KUAP_CURRENT) {
bedb4dbe CL	135	u32 kuap = current->thread.kuap;
88f8c080	136
bedb4dbe CL	137	if (unlikely(!kuap))
	138	return;
	139
	140	addr = kuap & 0xf0000000;
	141	end = kuap << 28;
	142	} else if (dir & KUAP_WRITE) {
	143	addr = (__force u32)to;
	144	end = min(addr + size, TASK_SIZE);
	145
	146	if (unlikely(addr >= TASK_SIZE \|\| !size))
	147	return;
	148	} else {
a68c31fc	149	return;
bedb4dbe	150	}
a68c31fc CL	151
	152	current->thread.kuap = 0;
	153	kuap_update_sr(mfsrin(addr) \| SR_KS, addr, end); /* set Ks */
	154	}
	155
3d7dfd63 CL	156	static inline unsigned long prevent_user_access_return(void)
	157	{
	158	unsigned long flags = current->thread.kuap;
	159	unsigned long addr = flags & 0xf0000000;
	160	unsigned long end = flags << 28;
	161	void __user to = (__force void __user )addr;
	162
	163	if (flags)
	164	prevent_user_access(to, to, end - addr, KUAP_READ_WRITE);
	165
	166	return flags;
	167	}
	168
	169	static inline void restore_user_access(unsigned long flags)
	170	{
	171	unsigned long addr = flags & 0xf0000000;
	172	unsigned long end = flags << 28;
	173	void __user to = (__force void __user )addr;
	174
	175	if (flags)
	176	allow_user_access(to, to, end - addr, KUAP_READ_WRITE);
	177	}
	178
6ec20aa2 CL	179	static inline bool
6ec20aa2 CL	180	bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
a68c31fc	181	{
6ec20aa2 CL	182	unsigned long begin = regs->kuap & 0xf0000000;
	183	unsigned long end = regs->kuap << 28;
	184
a68c31fc CL	185	if (!is_write)
	186	return false;
	187
6ec20aa2 CL	188	return WARN(address < begin \|\| address >= end,
6ec20aa2 CL	189	"Bug: write fault blocked by segment registers !");
a68c31fc CL	190	}
	191
	192	#endif /* CONFIG_PPC_KUAP */
	193
31ed2b13 CL	194	#endif /* __ASSEMBLY__ */
	195
	196	#endif /* _ASM_POWERPC_BOOK3S_32_KUP_H */