s390/memmove: Optimize backward copy case

memmove() copies byte wise for the backward copy case, when the mvc
instruction cannot be used. This is quite slow, but can be optimized
with the mvcrl instruction, which is available since z15.

Some numbers (measured on a shared z16 LPAR) show that the new
implementation is nearly always faster, except for the non realistic
one and two byte cases:

size    old   new
   1    2ns   3ns
   2    4ns   5ns
   4    5ns   5ns
   8    8ns   5ns
  16   12ns   6ns
  32    8ns   7ns
  64   15ns   7ns
 128   31ns   9ns
 256   64ns  10ns
 512  129ns  18ns
1024  250ns  19ns
2048  498ns  38ns

Reviewed-by: Juergen Christ <jchrist@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>

diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c
index cb65b8f5392a98f663500d837c76d25aa7c9bc3b..32e0e6b1e6239af45a9eb4fbc6ab219b4eaa2e4d 100644 (file)
--- a/arch/s390/lib/string.c
+++ b/arch/s390/lib/string.c
@@ -15,6 +15,7 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/export.h>
+#include <asm/facility.h>
 #include <asm/asm.h>
 
 #define SYMBOL_FUNCTION_ALIAS(alias, name)             \
@@ -51,8 +52,29 @@ noinstr void *__memmove(void *dest, const void *src, size_t n)
                                : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1)
                                : "memory");
                }
+               return dest;
+       }
+       /* Backward copy */
+       if (test_facility(61)) {
+               /* Use mvcrl instruction if available */
+               while (n >= 256) {
+                       asm volatile(
+                               "       lghi    %%r0,255\n"
+                               "       .insn   sse,0xe50a00000000,%[d],%[s]\n"
+                               : [d] "=Q" (*(d + n - 256))
+                               : [s] "Q" (*(s + n - 256))
+                               : "0", "memory");
+                       n -= 256;
+               }
+               if (n) {
+                       asm volatile(
+                               "       lgr     %%r0,%[n]\n"
+                               "       .insn   sse,0xe50a00000000,%[d],%[s]\n"
+                               : [d] "=Q" (*d)
+                               : [s] "Q" (*s), [n] "d" (n - 1)
+                               : "0", "memory");
+               }
        } else {
-               /* Backward copy */
                while (n--)
                        d[n] = s[n];
        }