s390/uaccess: Separate key uaccess functions

Implement separate raw_copy_to_user_key() and raw_copy_from_user_key()
functions, which allows to remove the open-coded operand access control
handling from the normal raw_copy_to_user() / raw_copy_from_user()
functions - they are simplified to use immediate instructions to load
hard-coded operand access control values into register zero, which saves
one instruction.

Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index f9ace938743ca37ff0408337cd0b6285cdea8a46..87833f50fa529acebe60038f64d45e6e4b9523ce 100644 (file)
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -22,28 +22,6 @@
 
 void debug_user_asce(int exit);
 
-union oac {
-       unsigned int val;
-       struct {
-               struct {
-                       unsigned short key : 4;
-                       unsigned short     : 4;
-                       unsigned short as  : 2;
-                       unsigned short     : 4;
-                       unsigned short k   : 1;
-                       unsigned short a   : 1;
-               } oac1;
-               struct {
-                       unsigned short key : 4;
-                       unsigned short     : 4;
-                       unsigned short as  : 2;
-                       unsigned short     : 4;
-                       unsigned short k   : 1;
-                       unsigned short a   : 1;
-               } oac2;
-       };
-};
-
 #ifdef CONFIG_KMSAN
 #define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory
 #else
@@ -51,28 +29,22 @@ union oac {
 #endif
 
 static uaccess_kmsan_or_inline __must_check unsigned long
-raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
+raw_copy_from_user(void *to, const void __user *from, unsigned long size)
 {
        unsigned long osize;
-       union oac spec = {
-               .oac2.key = key,
-               .oac2.as = PSW_BITS_AS_SECONDARY,
-               .oac2.k = 1,
-               .oac2.a = 1,
-       };
        int cc;
 
        while (1) {
                osize = size;
                asm_inline volatile(
-                       "       lr      %%r0,%[spec]\n"
+                       "       lhi     %%r0,%[spec]\n"
                        "0:     mvcos   %[to],%[from],%[size]\n"
                        "1:     nopr    %%r7\n"
                        CC_IPM(cc)
                        EX_TABLE_UA_MVCOS_FROM(0b, 0b)
                        EX_TABLE_UA_MVCOS_FROM(1b, 0b)
                        : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
-                       : [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
+                       : [spec] "I" (0x81), [from] "Q" (*(const char __user *)from)
                        : CC_CLOBBER_LIST("memory", "0"));
                if (likely(CC_TRANSFORM(cc) == 0))
                        return osize - size;
@@ -82,35 +54,23 @@ raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, un
        }
 }
 
-static __always_inline __must_check unsigned long
-raw_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-       return raw_copy_from_user_key(to, from, n, 0);
-}
-
 static uaccess_kmsan_or_inline __must_check unsigned long
-raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
+raw_copy_to_user(void __user *to, const void *from, unsigned long size)
 {
        unsigned long osize;
-       union oac spec = {
-               .oac1.key = key,
-               .oac1.as = PSW_BITS_AS_SECONDARY,
-               .oac1.k = 1,
-               .oac1.a = 1,
-       };
        int cc;
 
        while (1) {
                osize = size;
                asm_inline volatile(
-                       "       lr      %%r0,%[spec]\n"
+                       "       llilh   %%r0,%[spec]\n"
                        "0:     mvcos   %[to],%[from],%[size]\n"
                        "1:     nopr    %%r7\n"
                        CC_IPM(cc)
                        EX_TABLE_UA_MVCOS_TO(0b, 0b)
                        EX_TABLE_UA_MVCOS_TO(1b, 0b)
                        : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
-                       : [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
+                       : [spec] "I" (0x81), [from] "Q" (*(const char *)from)
                        : CC_CLOBBER_LIST("memory", "0"));
                if (likely(CC_TRANSFORM(cc) == 0))
                        return osize - size;
@@ -120,12 +80,6 @@ raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsi
        }
 }
 
-static __always_inline __must_check unsigned long
-raw_copy_to_user(void __user *to, const void *from, unsigned long n)
-{
-       return raw_copy_to_user_key(to, from, n, 0);
-}
-
 unsigned long __must_check
 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key);
 

diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index f977b7c37efc76221ace2aba1820ea90d655b1ef..2c85068aebfefa5c024eaadb8138d987c4f7ad65 100644 (file)
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -31,6 +31,60 @@ void debug_user_asce(int exit)
 }
 #endif /*CONFIG_DEBUG_ENTRY */
 
+union oac {
+       unsigned int val;
+       struct {
+               struct {
+                       unsigned short key : 4;
+                       unsigned short     : 4;
+                       unsigned short as  : 2;
+                       unsigned short     : 4;
+                       unsigned short k   : 1;
+                       unsigned short a   : 1;
+               } oac1;
+               struct {
+                       unsigned short key : 4;
+                       unsigned short     : 4;
+                       unsigned short as  : 2;
+                       unsigned short     : 4;
+                       unsigned short k   : 1;
+                       unsigned short a   : 1;
+               } oac2;
+       };
+};
+
+static uaccess_kmsan_or_inline __must_check unsigned long
+raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
+{
+       unsigned long osize;
+       union oac spec = {
+               .oac2.key = key,
+               .oac2.as = PSW_BITS_AS_SECONDARY,
+               .oac2.k = 1,
+               .oac2.a = 1,
+       };
+       int cc;
+
+       while (1) {
+               osize = size;
+               asm_inline volatile(
+                       "       lr      %%r0,%[spec]\n"
+                       "0:     mvcos   %[to],%[from],%[size]\n"
+                       "1:     nopr    %%r7\n"
+                       CC_IPM(cc)
+                       EX_TABLE_UA_MVCOS_FROM(0b, 0b)
+                       EX_TABLE_UA_MVCOS_FROM(1b, 0b)
+                       : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
+                       : [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
+                       : CC_CLOBBER_LIST("memory", "0"));
+               if (CC_TRANSFORM(cc) == 0)
+                       return osize - size;
+               size -= 4096;
+               to += 4096;
+               from += 4096;
+       }
+}
+
 unsigned long _copy_from_user_key(void *to, const void __user *from,
                                  unsigned long n, unsigned long key)
 {
@@ -48,6 +102,38 @@ unsigned long _copy_from_user_key(void *to, const void __user *from,
 }
 EXPORT_SYMBOL(_copy_from_user_key);
 
+static uaccess_kmsan_or_inline __must_check unsigned long
+raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
+{
+       unsigned long osize;
+       union oac spec = {
+               .oac1.key = key,
+               .oac1.as = PSW_BITS_AS_SECONDARY,
+               .oac1.k = 1,
+               .oac1.a = 1,
+       };
+       int cc;
+
+       while (1) {
+               osize = size;
+               asm_inline volatile(
+                       "       lr      %%r0,%[spec]\n"
+                       "0:     mvcos   %[to],%[from],%[size]\n"
+                       "1:     nopr    %%r7\n"
+                       CC_IPM(cc)
+                       EX_TABLE_UA_MVCOS_TO(0b, 0b)
+                       EX_TABLE_UA_MVCOS_TO(1b, 0b)
+                       : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
+                       : [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
+                       : CC_CLOBBER_LIST("memory", "0"));
+               if (CC_TRANSFORM(cc) == 0)
+                       return osize - size;
+               size -= 4096;
+               to += 4096;
+               from += 4096;
+       }
+}
+
 unsigned long _copy_to_user_key(void __user *to, const void *from,
                                unsigned long n, unsigned long key)
 {