xor: move generic implementations out of asm-generic/xor.h

Move the generic implementations from asm-generic/xor.h to
per-implementaion .c files in lib/raid.  This will build them
unconditionally even when an architecture forces a specific
implementation, but as we'll need at least one generic version for the
static_call optimization later on we'll pay that price.

Note that this would cause the second xor_block_8regs instance created by
arch/arm/lib/xor-neon.c to be generated instead of discarded as dead code,
so add a NO_TEMPLATE symbol to disable it for this case.

Link: https://lkml.kernel.org/r/20260327061704.3707577-11-hch@lst.de
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Tested-by: Eric Biggers <ebiggers@kernel.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: "Borislav Petkov (AMD)" <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Mason <clm@fb.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: David Sterba <dsterba@suse.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jason A. Donenfeld <jason@zx2c4.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Li Nan <linan122@huawei.com>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Magnus Lindholm <linmag7@gmail.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Richard Henderson <richard.henderson@linaro.org>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Song Liu <song@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Ted Ts'o <tytso@mit.edu>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c
index 282980b9bf2a2298432341c3ca6fe23922994273..b5be50567991bc41c74b0da68efb88e1d6633937 100644 (file)
--- a/arch/arm/lib/xor-neon.c
+++ b/arch/arm/lib/xor-neon.c
@@ -26,8 +26,8 @@ MODULE_LICENSE("GPL");
 #pragma GCC optimize "tree-vectorize"
 #endif
 
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#include <asm-generic/xor.h>
+#define NO_TEMPLATE
+#include "../../../lib/raid/xor/xor-8regs.c"
 
 struct xor_block_template const xor_block_neon_inner = {
        .name   = "__inner_neon__",

diff --git a/include/asm-generic/xor.h b/include/asm-generic/xor.h
index 79c0096aa9d957b6f278a1050dd685fb66847cc2..fc151fdc45ab09047b8647817bf900c2a0dc8e1b 100644 (file)
--- a/include/asm-generic/xor.h
+++ b/include/asm-generic/xor.h
@@ -5,726 +5,7 @@
  * Generic optimized RAID-5 checksumming functions.
  */
 
-#include <linux/prefetch.h>
-
-static void
-xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
-           const unsigned long * __restrict p2)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               p1[0] ^= p2[0];
-               p1[1] ^= p2[1];
-               p1[2] ^= p2[2];
-               p1[3] ^= p2[3];
-               p1[4] ^= p2[4];
-               p1[5] ^= p2[5];
-               p1[6] ^= p2[6];
-               p1[7] ^= p2[7];
-               p1 += 8;
-               p2 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
-           const unsigned long * __restrict p2,
-           const unsigned long * __restrict p3)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               p1[0] ^= p2[0] ^ p3[0];
-               p1[1] ^= p2[1] ^ p3[1];
-               p1[2] ^= p2[2] ^ p3[2];
-               p1[3] ^= p2[3] ^ p3[3];
-               p1[4] ^= p2[4] ^ p3[4];
-               p1[5] ^= p2[5] ^ p3[5];
-               p1[6] ^= p2[6] ^ p3[6];
-               p1[7] ^= p2[7] ^ p3[7];
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
-           const unsigned long * __restrict p2,
-           const unsigned long * __restrict p3,
-           const unsigned long * __restrict p4)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
-               p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
-               p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
-               p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
-               p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
-               p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
-               p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
-               p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
-           const unsigned long * __restrict p2,
-           const unsigned long * __restrict p3,
-           const unsigned long * __restrict p4,
-           const unsigned long * __restrict p5)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
-               p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
-               p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
-               p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
-               p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
-               p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
-               p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
-               p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-               p5 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
-            const unsigned long * __restrict p2)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
-            const unsigned long * __restrict p2,
-            const unsigned long * __restrict p3)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               d0 ^= p3[0];
-               d1 ^= p3[1];
-               d2 ^= p3[2];
-               d3 ^= p3[3];
-               d4 ^= p3[4];
-               d5 ^= p3[5];
-               d6 ^= p3[6];
-               d7 ^= p3[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
-            const unsigned long * __restrict p2,
-            const unsigned long * __restrict p3,
-            const unsigned long * __restrict p4)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               d0 ^= p3[0];
-               d1 ^= p3[1];
-               d2 ^= p3[2];
-               d3 ^= p3[3];
-               d4 ^= p3[4];
-               d5 ^= p3[5];
-               d6 ^= p3[6];
-               d7 ^= p3[7];
-               d0 ^= p4[0];
-               d1 ^= p4[1];
-               d2 ^= p4[2];
-               d3 ^= p4[3];
-               d4 ^= p4[4];
-               d5 ^= p4[5];
-               d6 ^= p4[6];
-               d7 ^= p4[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
-            const unsigned long * __restrict p2,
-            const unsigned long * __restrict p3,
-            const unsigned long * __restrict p4,
-            const unsigned long * __restrict p5)
-{
-       long lines = bytes / (sizeof (long)) / 8;
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               d0 ^= p3[0];
-               d1 ^= p3[1];
-               d2 ^= p3[2];
-               d3 ^= p3[3];
-               d4 ^= p3[4];
-               d5 ^= p3[5];
-               d6 ^= p3[6];
-               d7 ^= p3[7];
-               d0 ^= p4[0];
-               d1 ^= p4[1];
-               d2 ^= p4[2];
-               d3 ^= p4[3];
-               d4 ^= p4[4];
-               d5 ^= p4[5];
-               d6 ^= p4[6];
-               d7 ^= p4[7];
-               d0 ^= p5[0];
-               d1 ^= p5[1];
-               d2 ^= p5[2];
-               d3 ^= p5[3];
-               d4 ^= p5[4];
-               d5 ^= p5[5];
-               d6 ^= p5[6];
-               d7 ^= p5[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-               p5 += 8;
-       } while (--lines > 0);
-}
-
-static void
-xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
-             const unsigned long * __restrict p2)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-       prefetchw(p1);
-       prefetch(p2);
-
-       do {
-               prefetchw(p1+8);
-               prefetch(p2+8);
- once_more:
-               p1[0] ^= p2[0];
-               p1[1] ^= p2[1];
-               p1[2] ^= p2[2];
-               p1[3] ^= p2[3];
-               p1[4] ^= p2[4];
-               p1[5] ^= p2[5];
-               p1[6] ^= p2[6];
-               p1[7] ^= p2[7];
-               p1 += 8;
-               p2 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
-             const unsigned long * __restrict p2,
-             const unsigned long * __restrict p3)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-       prefetchw(p1);
-       prefetch(p2);
-       prefetch(p3);
-
-       do {
-               prefetchw(p1+8);
-               prefetch(p2+8);
-               prefetch(p3+8);
- once_more:
-               p1[0] ^= p2[0] ^ p3[0];
-               p1[1] ^= p2[1] ^ p3[1];
-               p1[2] ^= p2[2] ^ p3[2];
-               p1[3] ^= p2[3] ^ p3[3];
-               p1[4] ^= p2[4] ^ p3[4];
-               p1[5] ^= p2[5] ^ p3[5];
-               p1[6] ^= p2[6] ^ p3[6];
-               p1[7] ^= p2[7] ^ p3[7];
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
-             const unsigned long * __restrict p2,
-             const unsigned long * __restrict p3,
-             const unsigned long * __restrict p4)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-
-       prefetchw(p1);
-       prefetch(p2);
-       prefetch(p3);
-       prefetch(p4);
-
-       do {
-               prefetchw(p1+8);
-               prefetch(p2+8);
-               prefetch(p3+8);
-               prefetch(p4+8);
- once_more:
-               p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
-               p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
-               p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
-               p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
-               p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
-               p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
-               p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
-               p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
-             const unsigned long * __restrict p2,
-             const unsigned long * __restrict p3,
-             const unsigned long * __restrict p4,
-             const unsigned long * __restrict p5)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-
-       prefetchw(p1);
-       prefetch(p2);
-       prefetch(p3);
-       prefetch(p4);
-       prefetch(p5);
-
-       do {
-               prefetchw(p1+8);
-               prefetch(p2+8);
-               prefetch(p3+8);
-               prefetch(p4+8);
-               prefetch(p5+8);
- once_more:
-               p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
-               p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
-               p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
-               p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
-               p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
-               p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
-               p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
-               p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-               p5 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
-              const unsigned long * __restrict p2)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-
-       prefetchw(p1);
-       prefetch(p2);
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-               prefetchw(p1+8);
-               prefetch(p2+8);
- once_more:
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
-              const unsigned long * __restrict p2,
-              const unsigned long * __restrict p3)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-
-       prefetchw(p1);
-       prefetch(p2);
-       prefetch(p3);
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-               prefetchw(p1+8);
-               prefetch(p2+8);
-               prefetch(p3+8);
- once_more:
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               d0 ^= p3[0];
-               d1 ^= p3[1];
-               d2 ^= p3[2];
-               d3 ^= p3[3];
-               d4 ^= p3[4];
-               d5 ^= p3[5];
-               d6 ^= p3[6];
-               d7 ^= p3[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
-              const unsigned long * __restrict p2,
-              const unsigned long * __restrict p3,
-              const unsigned long * __restrict p4)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-
-       prefetchw(p1);
-       prefetch(p2);
-       prefetch(p3);
-       prefetch(p4);
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-               prefetchw(p1+8);
-               prefetch(p2+8);
-               prefetch(p3+8);
-               prefetch(p4+8);
- once_more:
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               d0 ^= p3[0];
-               d1 ^= p3[1];
-               d2 ^= p3[2];
-               d3 ^= p3[3];
-               d4 ^= p3[4];
-               d5 ^= p3[5];
-               d6 ^= p3[6];
-               d7 ^= p3[7];
-               d0 ^= p4[0];
-               d1 ^= p4[1];
-               d2 ^= p4[2];
-               d3 ^= p4[3];
-               d4 ^= p4[4];
-               d5 ^= p4[5];
-               d6 ^= p4[6];
-               d7 ^= p4[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static void
-xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
-              const unsigned long * __restrict p2,
-              const unsigned long * __restrict p3,
-              const unsigned long * __restrict p4,
-              const unsigned long * __restrict p5)
-{
-       long lines = bytes / (sizeof (long)) / 8 - 1;
-
-       prefetchw(p1);
-       prefetch(p2);
-       prefetch(p3);
-       prefetch(p4);
-       prefetch(p5);
-
-       do {
-               register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-               prefetchw(p1+8);
-               prefetch(p2+8);
-               prefetch(p3+8);
-               prefetch(p4+8);
-               prefetch(p5+8);
- once_more:
-               d0 = p1[0];     /* Pull the stuff into registers        */
-               d1 = p1[1];     /*  ... in bursts, if possible.         */
-               d2 = p1[2];
-               d3 = p1[3];
-               d4 = p1[4];
-               d5 = p1[5];
-               d6 = p1[6];
-               d7 = p1[7];
-               d0 ^= p2[0];
-               d1 ^= p2[1];
-               d2 ^= p2[2];
-               d3 ^= p2[3];
-               d4 ^= p2[4];
-               d5 ^= p2[5];
-               d6 ^= p2[6];
-               d7 ^= p2[7];
-               d0 ^= p3[0];
-               d1 ^= p3[1];
-               d2 ^= p3[2];
-               d3 ^= p3[3];
-               d4 ^= p3[4];
-               d5 ^= p3[5];
-               d6 ^= p3[6];
-               d7 ^= p3[7];
-               d0 ^= p4[0];
-               d1 ^= p4[1];
-               d2 ^= p4[2];
-               d3 ^= p4[3];
-               d4 ^= p4[4];
-               d5 ^= p4[5];
-               d6 ^= p4[6];
-               d7 ^= p4[7];
-               d0 ^= p5[0];
-               d1 ^= p5[1];
-               d2 ^= p5[2];
-               d3 ^= p5[3];
-               d4 ^= p5[4];
-               d5 ^= p5[5];
-               d6 ^= p5[6];
-               d7 ^= p5[7];
-               p1[0] = d0;     /* Store the result (in bursts)         */
-               p1[1] = d1;
-               p1[2] = d2;
-               p1[3] = d3;
-               p1[4] = d4;
-               p1[5] = d5;
-               p1[6] = d6;
-               p1[7] = d7;
-               p1 += 8;
-               p2 += 8;
-               p3 += 8;
-               p4 += 8;
-               p5 += 8;
-       } while (--lines > 0);
-       if (lines == 0)
-               goto once_more;
-}
-
-static struct xor_block_template xor_block_8regs = {
-       .name = "8regs",
-       .do_2 = xor_8regs_2,
-       .do_3 = xor_8regs_3,
-       .do_4 = xor_8regs_4,
-       .do_5 = xor_8regs_5,
-};
-
-static struct xor_block_template xor_block_32regs = {
-       .name = "32regs",
-       .do_2 = xor_32regs_2,
-       .do_3 = xor_32regs_3,
-       .do_4 = xor_32regs_4,
-       .do_5 = xor_32regs_5,
-};
-
-static struct xor_block_template xor_block_8regs_p __maybe_unused = {
-       .name = "8regs_prefetch",
-       .do_2 = xor_8regs_p_2,
-       .do_3 = xor_8regs_p_3,
-       .do_4 = xor_8regs_p_4,
-       .do_5 = xor_8regs_p_5,
-};
-
-static struct xor_block_template xor_block_32regs_p __maybe_unused = {
-       .name = "32regs_prefetch",
-       .do_2 = xor_32regs_p_2,
-       .do_3 = xor_32regs_p_3,
-       .do_4 = xor_32regs_p_4,
-       .do_5 = xor_32regs_p_5,
-};
+extern struct xor_block_template xor_block_8regs;
+extern struct xor_block_template xor_block_32regs;
+extern struct xor_block_template xor_block_8regs_p;
+extern struct xor_block_template xor_block_32regs_p;

diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 7bca0ce8e90ac02e1772b421f5b3a7bf410d6413..89a944c9f9909489748b54c5f1a7924a287dc84c 100644 (file)
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -3,3 +3,7 @@
 obj-$(CONFIG_XOR_BLOCKS)       += xor.o
 
 xor-y                          += xor-core.o
+xor-y                          += xor-8regs.o
+xor-y                          += xor-32regs.o
+xor-y                          += xor-8regs-prefetch.o
+xor-y                          += xor-32regs-prefetch.o

diff --git a/lib/raid/xor/xor-32regs-prefetch.c b/lib/raid/xor/xor-32regs-prefetch.c
new file mode 100644 (file)
index 0000000..8666c28
--- /dev/null
+++ b/lib/raid/xor/xor-32regs-prefetch.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/prefetch.h>
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
+              const unsigned long * __restrict p2)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+
+       prefetchw(p1);
+       prefetch(p2);
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+               prefetchw(p1+8);
+               prefetch(p2+8);
+ once_more:
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+static void
+xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
+              const unsigned long * __restrict p2,
+              const unsigned long * __restrict p3)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+
+       prefetchw(p1);
+       prefetch(p2);
+       prefetch(p3);
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+               prefetchw(p1+8);
+               prefetch(p2+8);
+               prefetch(p3+8);
+ once_more:
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               d0 ^= p3[0];
+               d1 ^= p3[1];
+               d2 ^= p3[2];
+               d3 ^= p3[3];
+               d4 ^= p3[4];
+               d5 ^= p3[5];
+               d6 ^= p3[6];
+               d7 ^= p3[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+static void
+xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
+              const unsigned long * __restrict p2,
+              const unsigned long * __restrict p3,
+              const unsigned long * __restrict p4)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+
+       prefetchw(p1);
+       prefetch(p2);
+       prefetch(p3);
+       prefetch(p4);
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+               prefetchw(p1+8);
+               prefetch(p2+8);
+               prefetch(p3+8);
+               prefetch(p4+8);
+ once_more:
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               d0 ^= p3[0];
+               d1 ^= p3[1];
+               d2 ^= p3[2];
+               d3 ^= p3[3];
+               d4 ^= p3[4];
+               d5 ^= p3[5];
+               d6 ^= p3[6];
+               d7 ^= p3[7];
+               d0 ^= p4[0];
+               d1 ^= p4[1];
+               d2 ^= p4[2];
+               d3 ^= p4[3];
+               d4 ^= p4[4];
+               d5 ^= p4[5];
+               d6 ^= p4[6];
+               d7 ^= p4[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+static void
+xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
+              const unsigned long * __restrict p2,
+              const unsigned long * __restrict p3,
+              const unsigned long * __restrict p4,
+              const unsigned long * __restrict p5)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+
+       prefetchw(p1);
+       prefetch(p2);
+       prefetch(p3);
+       prefetch(p4);
+       prefetch(p5);
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+               prefetchw(p1+8);
+               prefetch(p2+8);
+               prefetch(p3+8);
+               prefetch(p4+8);
+               prefetch(p5+8);
+ once_more:
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               d0 ^= p3[0];
+               d1 ^= p3[1];
+               d2 ^= p3[2];
+               d3 ^= p3[3];
+               d4 ^= p3[4];
+               d5 ^= p3[5];
+               d6 ^= p3[6];
+               d7 ^= p3[7];
+               d0 ^= p4[0];
+               d1 ^= p4[1];
+               d2 ^= p4[2];
+               d3 ^= p4[3];
+               d4 ^= p4[4];
+               d5 ^= p4[5];
+               d6 ^= p4[6];
+               d7 ^= p4[7];
+               d0 ^= p5[0];
+               d1 ^= p5[1];
+               d2 ^= p5[2];
+               d3 ^= p5[3];
+               d4 ^= p5[4];
+               d5 ^= p5[5];
+               d6 ^= p5[6];
+               d7 ^= p5[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+               p5 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+struct xor_block_template xor_block_32regs_p = {
+       .name = "32regs_prefetch",
+       .do_2 = xor_32regs_p_2,
+       .do_3 = xor_32regs_p_3,
+       .do_4 = xor_32regs_p_4,
+       .do_5 = xor_32regs_p_5,
+};

diff --git a/lib/raid/xor/xor-32regs.c b/lib/raid/xor/xor-32regs.c
new file mode 100644 (file)
index 0000000..58d4fac
--- /dev/null
+++ b/lib/raid/xor/xor-32regs.c
@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
+            const unsigned long * __restrict p2)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+       } while (--lines > 0);
+}
+
+static void
+xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
+            const unsigned long * __restrict p2,
+            const unsigned long * __restrict p3)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               d0 ^= p3[0];
+               d1 ^= p3[1];
+               d2 ^= p3[2];
+               d3 ^= p3[3];
+               d4 ^= p3[4];
+               d5 ^= p3[5];
+               d6 ^= p3[6];
+               d7 ^= p3[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+       } while (--lines > 0);
+}
+
+static void
+xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
+            const unsigned long * __restrict p2,
+            const unsigned long * __restrict p3,
+            const unsigned long * __restrict p4)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               d0 ^= p3[0];
+               d1 ^= p3[1];
+               d2 ^= p3[2];
+               d3 ^= p3[3];
+               d4 ^= p3[4];
+               d5 ^= p3[5];
+               d6 ^= p3[6];
+               d7 ^= p3[7];
+               d0 ^= p4[0];
+               d1 ^= p4[1];
+               d2 ^= p4[2];
+               d3 ^= p4[3];
+               d4 ^= p4[4];
+               d5 ^= p4[5];
+               d6 ^= p4[6];
+               d7 ^= p4[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+       } while (--lines > 0);
+}
+
+static void
+xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
+            const unsigned long * __restrict p2,
+            const unsigned long * __restrict p3,
+            const unsigned long * __restrict p4,
+            const unsigned long * __restrict p5)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               register long d0, d1, d2, d3, d4, d5, d6, d7;
+               d0 = p1[0];     /* Pull the stuff into registers        */
+               d1 = p1[1];     /*  ... in bursts, if possible.         */
+               d2 = p1[2];
+               d3 = p1[3];
+               d4 = p1[4];
+               d5 = p1[5];
+               d6 = p1[6];
+               d7 = p1[7];
+               d0 ^= p2[0];
+               d1 ^= p2[1];
+               d2 ^= p2[2];
+               d3 ^= p2[3];
+               d4 ^= p2[4];
+               d5 ^= p2[5];
+               d6 ^= p2[6];
+               d7 ^= p2[7];
+               d0 ^= p3[0];
+               d1 ^= p3[1];
+               d2 ^= p3[2];
+               d3 ^= p3[3];
+               d4 ^= p3[4];
+               d5 ^= p3[5];
+               d6 ^= p3[6];
+               d7 ^= p3[7];
+               d0 ^= p4[0];
+               d1 ^= p4[1];
+               d2 ^= p4[2];
+               d3 ^= p4[3];
+               d4 ^= p4[4];
+               d5 ^= p4[5];
+               d6 ^= p4[6];
+               d7 ^= p4[7];
+               d0 ^= p5[0];
+               d1 ^= p5[1];
+               d2 ^= p5[2];
+               d3 ^= p5[3];
+               d4 ^= p5[4];
+               d5 ^= p5[5];
+               d6 ^= p5[6];
+               d7 ^= p5[7];
+               p1[0] = d0;     /* Store the result (in bursts)         */
+               p1[1] = d1;
+               p1[2] = d2;
+               p1[3] = d3;
+               p1[4] = d4;
+               p1[5] = d5;
+               p1[6] = d6;
+               p1[7] = d7;
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+               p5 += 8;
+       } while (--lines > 0);
+}
+
+struct xor_block_template xor_block_32regs = {
+       .name = "32regs",
+       .do_2 = xor_32regs_2,
+       .do_3 = xor_32regs_3,
+       .do_4 = xor_32regs_4,
+       .do_5 = xor_32regs_5,
+};

diff --git a/lib/raid/xor/xor-8regs-prefetch.c b/lib/raid/xor/xor-8regs-prefetch.c
new file mode 100644 (file)
index 0000000..67061e3
--- /dev/null
+++ b/lib/raid/xor/xor-8regs-prefetch.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/prefetch.h>
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
+             const unsigned long * __restrict p2)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+       prefetchw(p1);
+       prefetch(p2);
+
+       do {
+               prefetchw(p1+8);
+               prefetch(p2+8);
+ once_more:
+               p1[0] ^= p2[0];
+               p1[1] ^= p2[1];
+               p1[2] ^= p2[2];
+               p1[3] ^= p2[3];
+               p1[4] ^= p2[4];
+               p1[5] ^= p2[5];
+               p1[6] ^= p2[6];
+               p1[7] ^= p2[7];
+               p1 += 8;
+               p2 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+static void
+xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
+             const unsigned long * __restrict p2,
+             const unsigned long * __restrict p3)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+       prefetchw(p1);
+       prefetch(p2);
+       prefetch(p3);
+
+       do {
+               prefetchw(p1+8);
+               prefetch(p2+8);
+               prefetch(p3+8);
+ once_more:
+               p1[0] ^= p2[0] ^ p3[0];
+               p1[1] ^= p2[1] ^ p3[1];
+               p1[2] ^= p2[2] ^ p3[2];
+               p1[3] ^= p2[3] ^ p3[3];
+               p1[4] ^= p2[4] ^ p3[4];
+               p1[5] ^= p2[5] ^ p3[5];
+               p1[6] ^= p2[6] ^ p3[6];
+               p1[7] ^= p2[7] ^ p3[7];
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+static void
+xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
+             const unsigned long * __restrict p2,
+             const unsigned long * __restrict p3,
+             const unsigned long * __restrict p4)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+
+       prefetchw(p1);
+       prefetch(p2);
+       prefetch(p3);
+       prefetch(p4);
+
+       do {
+               prefetchw(p1+8);
+               prefetch(p2+8);
+               prefetch(p3+8);
+               prefetch(p4+8);
+ once_more:
+               p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
+               p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
+               p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
+               p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
+               p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
+               p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
+               p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
+               p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+static void
+xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
+             const unsigned long * __restrict p2,
+             const unsigned long * __restrict p3,
+             const unsigned long * __restrict p4,
+             const unsigned long * __restrict p5)
+{
+       long lines = bytes / (sizeof (long)) / 8 - 1;
+
+       prefetchw(p1);
+       prefetch(p2);
+       prefetch(p3);
+       prefetch(p4);
+       prefetch(p5);
+
+       do {
+               prefetchw(p1+8);
+               prefetch(p2+8);
+               prefetch(p3+8);
+               prefetch(p4+8);
+               prefetch(p5+8);
+ once_more:
+               p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
+               p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
+               p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
+               p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
+               p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
+               p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
+               p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
+               p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+               p5 += 8;
+       } while (--lines > 0);
+       if (lines == 0)
+               goto once_more;
+}
+
+struct xor_block_template xor_block_8regs_p = {
+       .name = "8regs_prefetch",
+       .do_2 = xor_8regs_p_2,
+       .do_3 = xor_8regs_p_3,
+       .do_4 = xor_8regs_p_4,
+       .do_5 = xor_8regs_p_5,
+};

diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c
new file mode 100644 (file)
index 0000000..769f796
--- /dev/null
+++ b/lib/raid/xor/xor-8regs.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
+           const unsigned long * __restrict p2)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               p1[0] ^= p2[0];
+               p1[1] ^= p2[1];
+               p1[2] ^= p2[2];
+               p1[3] ^= p2[3];
+               p1[4] ^= p2[4];
+               p1[5] ^= p2[5];
+               p1[6] ^= p2[6];
+               p1[7] ^= p2[7];
+               p1 += 8;
+               p2 += 8;
+       } while (--lines > 0);
+}
+
+static void
+xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
+           const unsigned long * __restrict p2,
+           const unsigned long * __restrict p3)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               p1[0] ^= p2[0] ^ p3[0];
+               p1[1] ^= p2[1] ^ p3[1];
+               p1[2] ^= p2[2] ^ p3[2];
+               p1[3] ^= p2[3] ^ p3[3];
+               p1[4] ^= p2[4] ^ p3[4];
+               p1[5] ^= p2[5] ^ p3[5];
+               p1[6] ^= p2[6] ^ p3[6];
+               p1[7] ^= p2[7] ^ p3[7];
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+       } while (--lines > 0);
+}
+
+static void
+xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
+           const unsigned long * __restrict p2,
+           const unsigned long * __restrict p3,
+           const unsigned long * __restrict p4)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
+               p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
+               p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
+               p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
+               p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
+               p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
+               p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
+               p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+       } while (--lines > 0);
+}
+
+static void
+xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
+           const unsigned long * __restrict p2,
+           const unsigned long * __restrict p3,
+           const unsigned long * __restrict p4,
+           const unsigned long * __restrict p5)
+{
+       long lines = bytes / (sizeof (long)) / 8;
+
+       do {
+               p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
+               p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
+               p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
+               p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
+               p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
+               p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
+               p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
+               p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
+               p1 += 8;
+               p2 += 8;
+               p3 += 8;
+               p4 += 8;
+               p5 += 8;
+       } while (--lines > 0);
+}
+
+#ifndef NO_TEMPLATE
+struct xor_block_template xor_block_8regs = {
+       .name = "8regs",
+       .do_2 = xor_8regs_2,
+       .do_3 = xor_8regs_3,
+       .do_4 = xor_8regs_4,
+       .do_5 = xor_8regs_5,
+};
+#endif /* NO_TEMPLATE */