[thirdparty/linux.git] / arch / s390 / lib / uaccess.c

// SPDX-License-Identifier: GPL-2.0
/*
 *  Standard user space access functions based on mvcp/mvcs and doing
 *  interesting things in the secondary space mode.
 *
 *    Copyright IBM Corp. 2006,2014
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
 *		 Gerald Schaefer (gerald.schaefer@de.ibm.com)
 */

#include <linux/jump_label.h>
#include <linux/uaccess.h>
#include <linux/export.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <asm/mmu_context.h>
#include <asm/facility.h>

#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
static DEFINE_STATIC_KEY_FALSE(have_mvcos);

static int __init uaccess_init(void)
{
	if (test_facility(27))
		static_branch_enable(&have_mvcos);
	return 0;
}
early_initcall(uaccess_init);

static inline int copy_with_mvcos(void)
{
	if (static_branch_likely(&have_mvcos))
		return 1;
	return 0;
}
#else
static inline int copy_with_mvcos(void)
{
	return 1;
}
#endif

static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
						 unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x81UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
		"6: jz    4f\n"
		"1: algr  %0,%3\n"
		"   slgr  %1,%3\n"
		"   slgr  %2,%3\n"
		"   j     0b\n"
		"2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
		"   nr    %4,%3\n"	/* %4 = (ptr + 4095) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   5f\n"
		"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
		"7: slgr  %0,%4\n"
		"   j     5f\n"
		"4: slgr  %0,%0\n"
		"5:\n"
		EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
						unsigned long size)
{
	unsigned long tmp1, tmp2;

	load_kernel_asce();
	tmp1 = -256UL;
	asm volatile(
		"   sacf  0\n"
		"0: mvcp  0(%0,%2),0(%1),%3\n"
		"7: jz    5f\n"
		"1: algr  %0,%3\n"
		"   la    %1,256(%1)\n"
		"   la    %2,256(%2)\n"
		"2: mvcp  0(%0,%2),0(%1),%3\n"
		"8: jnz   1b\n"
		"   j     5f\n"
		"3: la    %4,255(%1)\n"	/* %4 = ptr + 255 */
		"   lghi  %3,-4096\n"
		"   nr    %4,%3\n"	/* %4 = (ptr + 255) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   6f\n"
		"4: mvcp  0(%4,%2),0(%1),%3\n"
		"9: slgr  %0,%4\n"
		"   j     6f\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
		EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: : "cc", "memory");
	return size;
}

unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
	if (copy_with_mvcos())
		return copy_from_user_mvcos(to, from, n);
	return copy_from_user_mvcp(to, from, n);
}
EXPORT_SYMBOL(raw_copy_from_user);

static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
					       unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x810000UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
		"6: jz    4f\n"
		"1: algr  %0,%3\n"
		"   slgr  %1,%3\n"
		"   slgr  %2,%3\n"
		"   j     0b\n"
		"2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
		"   nr    %4,%3\n"	/* %4 = (ptr + 4095) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   5f\n"
		"3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
		"7: slgr  %0,%4\n"
		"   j     5f\n"
		"4: slgr  %0,%0\n"
		"5:\n"
		EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
					      unsigned long size)
{
	unsigned long tmp1, tmp2;

	load_kernel_asce();
	tmp1 = -256UL;
	asm volatile(
		"   sacf  0\n"
		"0: mvcs  0(%0,%1),0(%2),%3\n"
		"7: jz    5f\n"
		"1: algr  %0,%3\n"
		"   la    %1,256(%1)\n"
		"   la    %2,256(%2)\n"
		"2: mvcs  0(%0,%1),0(%2),%3\n"
		"8: jnz   1b\n"
		"   j     5f\n"
		"3: la    %4,255(%1)\n" /* %4 = ptr + 255 */
		"   lghi  %3,-4096\n"
		"   nr    %4,%3\n"	/* %4 = (ptr + 255) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   6f\n"
		"4: mvcs  0(%4,%1),0(%2),%3\n"
		"9: slgr  %0,%4\n"
		"   j     6f\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
		EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: : "cc", "memory");
	return size;
}

unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
	if (copy_with_mvcos())
		return copy_to_user_mvcos(to, from, n);
	return copy_to_user_mvcs(to, from, n);
}
EXPORT_SYMBOL(raw_copy_to_user);

static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
					       unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x810081UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	/* FIXME: copy with reduced length. */
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
		"   jz	  2f\n"
		"1: algr  %0,%3\n"
		"   slgr  %1,%3\n"
		"   slgr  %2,%3\n"
		"   j	  0b\n"
		"2:slgr  %0,%0\n"
		"3: \n"
		EX_TABLE(0b,3b)
		: "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from,
					     unsigned long size)
{
	unsigned long tmp1;

	load_kernel_asce();
	asm volatile(
		"   sacf  256\n"
		"   aghi  %0,-1\n"
		"   jo	  5f\n"
		"   bras  %3,3f\n"
		"0: aghi  %0,257\n"
		"1: mvc	  0(1,%1),0(%2)\n"
		"   la	  %1,1(%1)\n"
		"   la	  %2,1(%2)\n"
		"   aghi  %0,-1\n"
		"   jnz	  1b\n"
		"   j	  5f\n"
		"2: mvc	  0(256,%1),0(%2)\n"
		"   la	  %1,256(%1)\n"
		"   la	  %2,256(%2)\n"
		"3: aghi  %0,-256\n"
		"   jnm	  2b\n"
		"4: ex	  %0,1b-0b(%3)\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
		: "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
		: : "cc", "memory");
	return size;
}

unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
{
	if (copy_with_mvcos())
		return copy_in_user_mvcos(to, from, n);
	return copy_in_user_mvc(to, from, n);
}
EXPORT_SYMBOL(raw_copy_in_user);

static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x810000UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
		"   jz	  4f\n"
		"1: algr  %0,%2\n"
		"   slgr  %1,%2\n"
		"   j	  0b\n"
		"2: la	  %3,4095(%1)\n"/* %4 = to + 4095 */
		"   nr	  %3,%2\n"	/* %4 = (to + 4095) & -4096 */
		"   slgr  %3,%1\n"
		"   clgr  %0,%3\n"	/* copy crosses next page boundary? */
		"   jnh	  5f\n"
		"3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
		"   slgr  %0,%3\n"
		"   j	  5f\n"
		"4: slgr  %0,%0\n"
		"5:\n"
		EX_TABLE(0b,2b) EX_TABLE(3b,5b)
		: "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
		: "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
{
	unsigned long tmp1, tmp2;

	load_kernel_asce();
	asm volatile(
		"   sacf  256\n"
		"   aghi  %0,-1\n"
		"   jo    5f\n"
		"   bras  %3,3f\n"
		"   xc    0(1,%1),0(%1)\n"
		"0: aghi  %0,257\n"
		"   la    %2,255(%1)\n" /* %2 = ptr + 255 */
		"   srl   %2,12\n"
		"   sll   %2,12\n"	/* %2 = (ptr + 255) & -4096 */
		"   slgr  %2,%1\n"
		"   clgr  %0,%2\n"	/* clear crosses next page boundary? */
		"   jnh   5f\n"
		"   aghi  %2,-1\n"
		"1: ex    %2,0(%3)\n"
		"   aghi  %2,1\n"
		"   slgr  %0,%2\n"
		"   j     5f\n"
		"2: xc    0(256,%1),0(%1)\n"
		"   la    %1,256(%1)\n"
		"3: aghi  %0,-256\n"
		"   jnm   2b\n"
		"4: ex    %0,0(%3)\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
		: "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
		: : "cc", "memory");
	return size;
}

unsigned long __clear_user(void __user *to, unsigned long size)
{
	if (copy_with_mvcos())
			return clear_user_mvcos(to, size);
	return clear_user_xc(to, size);
}
EXPORT_SYMBOL(__clear_user);

static inline unsigned long strnlen_user_srst(const char __user *src,
					      unsigned long size)
{
	register unsigned long reg0 asm("0") = 0;
	unsigned long tmp1, tmp2;

	asm volatile(
		"   la    %2,0(%1)\n"
		"   la    %3,0(%0,%1)\n"
		"   slgr  %0,%0\n"
		"   sacf  256\n"
		"0: srst  %3,%2\n"
		"   jo    0b\n"
		"   la    %0,1(%3)\n"	/* strnlen_user results includes \0 */
		"   slgr  %0,%1\n"
		"1: sacf  768\n"
		EX_TABLE(0b,1b)
		: "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

unsigned long __strnlen_user(const char __user *src, unsigned long size)
{
	if (unlikely(!size))
		return 0;
	load_kernel_asce();
	return strnlen_user_srst(src, size);
}
EXPORT_SYMBOL(__strnlen_user);

long __strncpy_from_user(char *dst, const char __user *src, long size)
{
	size_t done, len, offset, len_str;

	if (unlikely(size <= 0))
		return 0;
	done = 0;
	do {
		offset = (size_t)src & (L1_CACHE_BYTES - 1);
		len = min(size - done, L1_CACHE_BYTES - offset);
		if (copy_from_user(dst, src, len))
			return -EFAULT;
		len_str = strnlen(dst, len);
		done += len_str;
		src += len_str;
		dst += len_str;
	} while ((len_str == len) && (done < size));
	return done;
}
EXPORT_SYMBOL(__strncpy_from_user);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
457f2180 HC	2	/*
	3	* Standard user space access functions based on mvcp/mvcs and doing
	4	* interesting things in the secondary space mode.
	5	*
	6	* Copyright IBM Corp. 2006,2014
	7	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
	8	* Gerald Schaefer (gerald.schaefer@de.ibm.com)
	9	*/
	10
	11	#include <linux/jump_label.h>
	12	#include <linux/uaccess.h>
	13	#include <linux/export.h>
	14	#include <linux/errno.h>
	15	#include <linux/mm.h>
	16	#include <asm/mmu_context.h>
	17	#include <asm/facility.h>
	18
d66bf801	19	#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
ed79e946	20	static DEFINE_STATIC_KEY_FALSE(have_mvcos);
457f2180	21
d66bf801 MS	22	static int __init uaccess_init(void)
	23	{
	24	if (test_facility(27))
	25	static_branch_enable(&have_mvcos);
	26	return 0;
	27	}
	28	early_initcall(uaccess_init);
	29
	30	static inline int copy_with_mvcos(void)
	31	{
	32	if (static_branch_likely(&have_mvcos))
	33	return 1;
	34	return 0;
	35	}
	36	#else
	37	static inline int copy_with_mvcos(void)
	38	{
	39	return 1;
	40	}
	41	#endif
	42
457f2180 HC	43	static inline unsigned long copy_from_user_mvcos(void x, const void __user ptr,
	44	unsigned long size)
	45	{
	46	register unsigned long reg0 asm("0") = 0x81UL;
	47	unsigned long tmp1, tmp2;
	48
	49	tmp1 = -4096UL;
	50	asm volatile(
	51	"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
37096003	52	"6: jz 4f\n"
5a79859a HC	53	"1: algr %0,%3\n"
	54	" slgr %1,%3\n"
	55	" slgr %2,%3\n"
457f2180 HC	56	" j 0b\n"
	57	"2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
	58	" nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
5a79859a HC	59	" slgr %4,%1\n"
5a79859a HC	60	" clgr %0,%4\n" /* copy crosses next page boundary? */
37096003	61	" jnh 5f\n"
457f2180	62	"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
37096003 AV	63	"7: slgr %0,%4\n"
	64	" j 5f\n"
	65	"4: slgr %0,%0\n"
	66	"5:\n"
	67	EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
457f2180 HC	68	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	69	: "d" (reg0) : "cc", "memory");
	70	return size;
	71	}
	72
	73	static inline unsigned long copy_from_user_mvcp(void x, const void __user ptr,
	74	unsigned long size)
	75	{
	76	unsigned long tmp1, tmp2;
	77
beef560b	78	load_kernel_asce();
457f2180 HC	79	tmp1 = -256UL;
	80	asm volatile(
	81	" sacf 0\n"
	82	"0: mvcp 0(%0,%2),0(%1),%3\n"
37096003	83	"7: jz 5f\n"
5a79859a	84	"1: algr %0,%3\n"
457f2180 HC	85	" la %1,256(%1)\n"
	86	" la %2,256(%2)\n"
	87	"2: mvcp 0(%0,%2),0(%1),%3\n"
37096003 AV	88	"8: jnz 1b\n"
37096003 AV	89	" j 5f\n"
457f2180	90	"3: la %4,255(%1)\n" /* %4 = ptr + 255 */
5a79859a	91	" lghi %3,-4096\n"
457f2180	92	" nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
5a79859a HC	93	" slgr %4,%1\n"
5a79859a HC	94	" clgr %0,%4\n" /* copy crosses next page boundary? */
37096003	95	" jnh 6f\n"
457f2180	96	"4: mvcp 0(%4,%2),0(%1),%3\n"
37096003 AV	97	"9: slgr %0,%4\n"
	98	" j 6f\n"
	99	"5: slgr %0,%0\n"
	100	"6: sacf 768\n"
	101	EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
	102	EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
457f2180 HC	103	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	104	: : "cc", "memory");
	105	return size;
	106	}
	107
37096003	108	unsigned long raw_copy_from_user(void to, const void __user from, unsigned long n)
457f2180	109	{
d66bf801	110	if (copy_with_mvcos())
457f2180 HC	111	return copy_from_user_mvcos(to, from, n);
	112	return copy_from_user_mvcp(to, from, n);
	113	}
37096003	114	EXPORT_SYMBOL(raw_copy_from_user);
457f2180 HC	115
	116	static inline unsigned long copy_to_user_mvcos(void __user ptr, const void x,
	117	unsigned long size)
	118	{
	119	register unsigned long reg0 asm("0") = 0x810000UL;
	120	unsigned long tmp1, tmp2;
	121
	122	tmp1 = -4096UL;
	123	asm volatile(
	124	"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
	125	"6: jz 4f\n"
5a79859a HC	126	"1: algr %0,%3\n"
	127	" slgr %1,%3\n"
	128	" slgr %2,%3\n"
457f2180 HC	129	" j 0b\n"
	130	"2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
	131	" nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
5a79859a HC	132	" slgr %4,%1\n"
5a79859a HC	133	" clgr %0,%4\n" /* copy crosses next page boundary? */
457f2180 HC	134	" jnh 5f\n"
457f2180 HC	135	"3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
5a79859a	136	"7: slgr %0,%4\n"
457f2180	137	" j 5f\n"
5a79859a	138	"4: slgr %0,%0\n"
457f2180 HC	139	"5:\n"
	140	EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
	141	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	142	: "d" (reg0) : "cc", "memory");
	143	return size;
	144	}
	145
	146	static inline unsigned long copy_to_user_mvcs(void __user ptr, const void x,
	147	unsigned long size)
	148	{
	149	unsigned long tmp1, tmp2;
	150
beef560b	151	load_kernel_asce();
457f2180 HC	152	tmp1 = -256UL;
	153	asm volatile(
	154	" sacf 0\n"
	155	"0: mvcs 0(%0,%1),0(%2),%3\n"
	156	"7: jz 5f\n"
5a79859a	157	"1: algr %0,%3\n"
457f2180 HC	158	" la %1,256(%1)\n"
	159	" la %2,256(%2)\n"
	160	"2: mvcs 0(%0,%1),0(%2),%3\n"
	161	"8: jnz 1b\n"
	162	" j 5f\n"
	163	"3: la %4,255(%1)\n" /* %4 = ptr + 255 */
5a79859a	164	" lghi %3,-4096\n"
457f2180	165	" nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
5a79859a HC	166	" slgr %4,%1\n"
5a79859a HC	167	" clgr %0,%4\n" /* copy crosses next page boundary? */
457f2180 HC	168	" jnh 6f\n"
457f2180 HC	169	"4: mvcs 0(%4,%1),0(%2),%3\n"
5a79859a	170	"9: slgr %0,%4\n"
457f2180	171	" j 6f\n"
5a79859a	172	"5: slgr %0,%0\n"
457f2180 HC	173	"6: sacf 768\n"
	174	EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
	175	EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
	176	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	177	: : "cc", "memory");
	178	return size;
	179	}
	180
37096003	181	unsigned long raw_copy_to_user(void __user to, const void from, unsigned long n)
457f2180	182	{
d66bf801	183	if (copy_with_mvcos())
457f2180 HC	184	return copy_to_user_mvcos(to, from, n);
	185	return copy_to_user_mvcs(to, from, n);
	186	}
37096003	187	EXPORT_SYMBOL(raw_copy_to_user);
457f2180 HC	188
	189	static inline unsigned long copy_in_user_mvcos(void __user to, const void __user from,
	190	unsigned long size)
	191	{
	192	register unsigned long reg0 asm("0") = 0x810081UL;
	193	unsigned long tmp1, tmp2;
	194
	195	tmp1 = -4096UL;
	196	/* FIXME: copy with reduced length. */
	197	asm volatile(
	198	"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
	199	" jz 2f\n"
5a79859a HC	200	"1: algr %0,%3\n"
	201	" slgr %1,%3\n"
	202	" slgr %2,%3\n"
457f2180	203	" j 0b\n"
5a79859a	204	"2:slgr %0,%0\n"
457f2180 HC	205	"3: \n"
	206	EX_TABLE(0b,3b)
	207	: "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
	208	: "d" (reg0) : "cc", "memory");
	209	return size;
	210	}
	211
	212	static inline unsigned long copy_in_user_mvc(void __user to, const void __user from,
	213	unsigned long size)
	214	{
	215	unsigned long tmp1;
	216
beef560b	217	load_kernel_asce();
457f2180 HC	218	asm volatile(
457f2180 HC	219	" sacf 256\n"
5a79859a	220	" aghi %0,-1\n"
457f2180 HC	221	" jo 5f\n"
457f2180 HC	222	" bras %3,3f\n"
5a79859a	223	"0: aghi %0,257\n"
457f2180 HC	224	"1: mvc 0(1,%1),0(%2)\n"
	225	" la %1,1(%1)\n"
	226	" la %2,1(%2)\n"
5a79859a	227	" aghi %0,-1\n"
457f2180 HC	228	" jnz 1b\n"
	229	" j 5f\n"
	230	"2: mvc 0(256,%1),0(%2)\n"
	231	" la %1,256(%1)\n"
	232	" la %2,256(%2)\n"
5a79859a	233	"3: aghi %0,-256\n"
457f2180 HC	234	" jnm 2b\n"
457f2180 HC	235	"4: ex %0,1b-0b(%3)\n"
5a79859a	236	"5: slgr %0,%0\n"
457f2180 HC	237	"6: sacf 768\n"
	238	EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
	239	: "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
	240	: : "cc", "memory");
	241	return size;
	242	}
	243
37096003	244	unsigned long raw_copy_in_user(void __user to, const void __user from, unsigned long n)
457f2180	245	{
d66bf801	246	if (copy_with_mvcos())
457f2180 HC	247	return copy_in_user_mvcos(to, from, n);
	248	return copy_in_user_mvc(to, from, n);
	249	}
37096003	250	EXPORT_SYMBOL(raw_copy_in_user);
457f2180 HC	251
	252	static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
	253	{
	254	register unsigned long reg0 asm("0") = 0x810000UL;
	255	unsigned long tmp1, tmp2;
	256
	257	tmp1 = -4096UL;
	258	asm volatile(
	259	"0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
	260	" jz 4f\n"
5a79859a HC	261	"1: algr %0,%2\n"
5a79859a HC	262	" slgr %1,%2\n"
457f2180 HC	263	" j 0b\n"
	264	"2: la %3,4095(%1)\n"/* %4 = to + 4095 */
	265	" nr %3,%2\n" /* %4 = (to + 4095) & -4096 */
5a79859a HC	266	" slgr %3,%1\n"
5a79859a HC	267	" clgr %0,%3\n" /* copy crosses next page boundary? */
457f2180 HC	268	" jnh 5f\n"
457f2180 HC	269	"3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
5a79859a	270	" slgr %0,%3\n"
457f2180	271	" j 5f\n"
b8ac5e2f	272	"4: slgr %0,%0\n"
457f2180 HC	273	"5:\n"
	274	EX_TABLE(0b,2b) EX_TABLE(3b,5b)
	275	: "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
	276	: "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
	277	return size;
	278	}
	279
	280	static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
	281	{
	282	unsigned long tmp1, tmp2;
	283
beef560b	284	load_kernel_asce();
457f2180 HC	285	asm volatile(
457f2180 HC	286	" sacf 256\n"
5a79859a	287	" aghi %0,-1\n"
457f2180 HC	288	" jo 5f\n"
	289	" bras %3,3f\n"
	290	" xc 0(1,%1),0(%1)\n"
5a79859a	291	"0: aghi %0,257\n"
457f2180 HC	292	" la %2,255(%1)\n" /* %2 = ptr + 255 */
	293	" srl %2,12\n"
	294	" sll %2,12\n" /* %2 = (ptr + 255) & -4096 */
5a79859a HC	295	" slgr %2,%1\n"
5a79859a HC	296	" clgr %0,%2\n" /* clear crosses next page boundary? */
457f2180	297	" jnh 5f\n"
5a79859a	298	" aghi %2,-1\n"
457f2180	299	"1: ex %2,0(%3)\n"
5a79859a HC	300	" aghi %2,1\n"
5a79859a HC	301	" slgr %0,%2\n"
457f2180 HC	302	" j 5f\n"
	303	"2: xc 0(256,%1),0(%1)\n"
	304	" la %1,256(%1)\n"
5a79859a	305	"3: aghi %0,-256\n"
457f2180 HC	306	" jnm 2b\n"
457f2180 HC	307	"4: ex %0,0(%3)\n"
5a79859a	308	"5: slgr %0,%0\n"
457f2180 HC	309	"6: sacf 768\n"
	310	EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
	311	: "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
	312	: : "cc", "memory");
	313	return size;
	314	}
	315
	316	unsigned long __clear_user(void __user *to, unsigned long size)
	317	{
d66bf801	318	if (copy_with_mvcos())
457f2180 HC	319	return clear_user_mvcos(to, size);
	320	return clear_user_xc(to, size);
	321	}
	322	EXPORT_SYMBOL(__clear_user);
	323
	324	static inline unsigned long strnlen_user_srst(const char __user *src,
	325	unsigned long size)
	326	{
	327	register unsigned long reg0 asm("0") = 0;
	328	unsigned long tmp1, tmp2;
	329
457f2180 HC	330	asm volatile(
	331	" la %2,0(%1)\n"
	332	" la %3,0(%0,%1)\n"
5a79859a	333	" slgr %0,%0\n"
457f2180 HC	334	" sacf 256\n"
	335	"0: srst %3,%2\n"
	336	" jo 0b\n"
	337	" la %0,1(%3)\n" /* strnlen_user results includes \0 */
5a79859a	338	" slgr %0,%1\n"
457f2180 HC	339	"1: sacf 768\n"
	340	EX_TABLE(0b,1b)
	341	: "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2)
	342	: "d" (reg0) : "cc", "memory");
	343	return size;
	344	}
	345
	346	unsigned long __strnlen_user(const char __user *src, unsigned long size)
	347	{
fa255f51 HC	348	if (unlikely(!size))
fa255f51 HC	349	return 0;
beef560b	350	load_kernel_asce();
457f2180 HC	351	return strnlen_user_srst(src, size);
	352	}
	353	EXPORT_SYMBOL(__strnlen_user);
	354
	355	long __strncpy_from_user(char dst, const char __user src, long size)
	356	{
	357	size_t done, len, offset, len_str;
	358
	359	if (unlikely(size <= 0))
	360	return 0;
	361	done = 0;
	362	do {
f5c8b960 HC	363	offset = (size_t)src & (L1_CACHE_BYTES - 1);
f5c8b960 HC	364	len = min(size - done, L1_CACHE_BYTES - offset);
457f2180 HC	365	if (copy_from_user(dst, src, len))
	366	return -EFAULT;
	367	len_str = strnlen(dst, len);
	368	done += len_str;
	369	src += len_str;
	370	dst += len_str;
	371	} while ((len_str == len) && (done < size));
	372	return done;
	373	}
	374	EXPORT_SYMBOL(__strncpy_from_user);