s390/uaccess: Inline __clear_user()

Rework __clear_user() similar to raw_copy_from_user() / raw_copy_to_user()
and inline the function saving the overhead of branches.

Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index b5712d690a93dbe618865c2a35f9caa9439d31c1..f30ee6ddfece244a3b65685ac748c4c7e7a5a60f 100644 (file)
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -13,6 +13,7 @@
 /*
  * User space memory access functions
  */
+#include <linux/pgtable.h>
 #include <asm/asm-extable.h>
 #include <asm/processor.h>
 #include <asm/extable.h>
@@ -362,12 +363,34 @@ long __must_check strncpy_from_user(char *dst, const char __user *src, long coun
 
 long __must_check strnlen_user(const char __user *src, long count);
 
-/*
- * Zero Userspace
- */
-unsigned long __must_check __clear_user(void __user *to, unsigned long size);
+static uaccess_kmsan_or_inline __must_check unsigned long
+__clear_user(void __user *to, unsigned long size)
+{
+       unsigned long osize;
+       int cc;
+
+       while (1) {
+               osize = size;
+               asm_inline volatile(
+                       "       llilh   %%r0,%[spec]\n"
+                       "0:     mvcos   %[to],%[from],%[size]\n"
+                       "1:     nopr    %%r7\n"
+                       CC_IPM(cc)
+                       EX_TABLE_UA_MVCOS_TO(0b, 0b)
+                       EX_TABLE_UA_MVCOS_TO(1b, 0b)
+                       : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
+                       : [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page)
+                       : CC_CLOBBER_LIST("memory", "0"));
+               if (__builtin_constant_p(osize) && osize <= 4096)
+                       return osize - size;
+               if (CC_TRANSFORM(cc) == 0)
+                       return osize - size;
+               size -= 4096;
+               to += 4096;
+       }
+}
 
-static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
+static __always_inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
 {
        might_fault();
        return __clear_user(to, n);

diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index 2c85068aebfefa5c024eaadb8138d987c4f7ad65..cec20db884795f085032537f62c32167a75dfed2 100644 (file)
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -144,39 +144,3 @@ unsigned long _copy_to_user_key(void __user *to, const void *from,
        return raw_copy_to_user_key(to, from, n, key);
 }
 EXPORT_SYMBOL(_copy_to_user_key);
-
-unsigned long __clear_user(void __user *to, unsigned long size)
-{
-       unsigned long rem;
-       union oac spec = {
-               .oac1.as = PSW_BITS_AS_SECONDARY,
-               .oac1.a = 1,
-       };
-
-       asm volatile(
-               "       lr      0,%[spec]\n"
-               "0:     mvcos   0(%[to]),0(%[zeropg]),%[size]\n"
-               "1:     jz      5f\n"
-               "       algr    %[size],%[val]\n"
-               "       slgr    %[to],%[val]\n"
-               "       j       0b\n"
-               "2:     la      %[rem],4095(%[to])\n"   /* rem = to + 4095 */
-               "       nr      %[rem],%[val]\n"        /* rem = (to + 4095) & -4096 */
-               "       slgr    %[rem],%[to]\n"
-               "       clgr    %[size],%[rem]\n"       /* copy crosses next page boundary? */
-               "       jnh     6f\n"
-               "3:     mvcos   0(%[to]),0(%[zeropg]),%[rem]\n"
-               "4:     slgr    %[size],%[rem]\n"
-               "       j       6f\n"
-               "5:     slgr    %[size],%[size]\n"
-               "6:\n"
-               EX_TABLE(0b, 2b)
-               EX_TABLE(1b, 2b)
-               EX_TABLE(3b, 6b)
-               EX_TABLE(4b, 6b)
-               : [size] "+&a" (size), [to] "+&a" (to), [rem] "=&a" (rem)
-               : [val] "a" (-4096UL), [zeropg] "a" (empty_zero_page), [spec] "d" (spec.val)
-               : "cc", "memory", "0");
-       return size;
-}
-EXPORT_SYMBOL(__clear_user);