1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2016 Intel Corporation
5 * Author: Gayatri Kammela <gayatri.kammela@intel.com>
6 * Author: Megha Dey <megha.dey@linux.intel.com>
9 #include <linux/raid/pq.h>
12 static int raid6_has_avx512(void)
14 return boot_cpu_has(X86_FEATURE_AVX2) &&
15 boot_cpu_has(X86_FEATURE_AVX) &&
16 boot_cpu_has(X86_FEATURE_AVX512F) &&
17 boot_cpu_has(X86_FEATURE_AVX512BW) &&
18 boot_cpu_has(X86_FEATURE_AVX512VL) &&
19 boot_cpu_has(X86_FEATURE_AVX512DQ);
22 static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
23 int failb, void **ptrs)
26 const u8 *pbmul; /* P multiplier table for B data */
27 const u8 *qmul; /* Q multiplier table (for both) */
30 p = (u8 *)ptrs[disks-2];
31 q = (u8 *)ptrs[disks-1];
34 * Compute syndrome with zero for the missing data pages
35 * Use the dead data pages as temporary storage for
39 dp = (u8 *)ptrs[faila];
40 ptrs[faila] = raid6_get_zero_page();
42 dq = (u8 *)ptrs[failb];
43 ptrs[failb] = raid6_get_zero_page();
46 raid6_call.gen_syndrome(disks, bytes, ptrs);
48 /* Restore pointer table */
54 /* Now, pick the proper data tables */
55 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
56 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
62 asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
66 asm volatile("vmovdqa64 %0, %%zmm1\n\t"
67 "vmovdqa64 %1, %%zmm9\n\t"
68 "vmovdqa64 %2, %%zmm0\n\t"
69 "vmovdqa64 %3, %%zmm8\n\t"
70 "vpxorq %4, %%zmm1, %%zmm1\n\t"
71 "vpxorq %5, %%zmm9, %%zmm9\n\t"
72 "vpxorq %6, %%zmm0, %%zmm0\n\t"
73 "vpxorq %7, %%zmm8, %%zmm8"
75 : "m" (q[0]), "m" (q[64]), "m" (p[0]),
76 "m" (p[64]), "m" (dq[0]), "m" (dq[64]),
77 "m" (dp[0]), "m" (dp[64]));
86 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
87 "vbroadcasti64x2 %1, %%zmm5"
89 : "m" (qmul[0]), "m" (qmul[16]));
91 asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
92 "vpsraw $4, %%zmm9, %%zmm12\n\t"
93 "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
94 "vpandq %%zmm7, %%zmm9, %%zmm9\n\t"
95 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
96 "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
97 "vpshufb %%zmm9, %%zmm4, %%zmm14\n\t"
98 "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
99 "vpshufb %%zmm12, %%zmm5, %%zmm15\n\t"
100 "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
101 "vpxorq %%zmm14, %%zmm15, %%zmm15\n\t"
102 "vpxorq %%zmm4, %%zmm5, %%zmm5"
111 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
112 "vbroadcasti64x2 %1, %%zmm1\n\t"
113 "vpsraw $4, %%zmm0, %%zmm2\n\t"
114 "vpsraw $4, %%zmm8, %%zmm6\n\t"
115 "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
116 "vpandq %%zmm7, %%zmm8, %%zmm14\n\t"
117 "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
118 "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
119 "vpshufb %%zmm14, %%zmm4, %%zmm12\n\t"
120 "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
121 "vpshufb %%zmm6, %%zmm1, %%zmm13\n\t"
122 "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
123 "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t"
124 "vpxorq %%zmm12, %%zmm13, %%zmm13"
126 : "m" (pbmul[0]), "m" (pbmul[16]));
132 asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
133 "vpxorq %%zmm15, %%zmm13, %%zmm13"
139 * 13 = db[64] = DQ[64]
141 asm volatile("vmovdqa64 %%zmm1, %0\n\t"
142 "vmovdqa64 %%zmm13,%1\n\t"
143 "vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
144 "vpxorq %%zmm13, %%zmm8, %%zmm8"
146 : "m" (dq[0]), "m" (dq[64]));
148 asm volatile("vmovdqa64 %%zmm0, %0\n\t"
149 "vmovdqa64 %%zmm8, %1"
151 : "m" (dp[0]), "m" (dp[64]));
159 asm volatile("vmovdqa64 %0, %%zmm1\n\t"
160 "vmovdqa64 %1, %%zmm0\n\t"
161 "vpxorq %2, %%zmm1, %%zmm1\n\t"
162 "vpxorq %3, %%zmm0, %%zmm0"
164 : "m" (*q), "m" (*p), "m"(*dq), "m" (*dp));
166 /* 1 = dq ^ q; 0 = dp ^ p */
168 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
169 "vbroadcasti64x2 %1, %%zmm5"
171 : "m" (qmul[0]), "m" (qmul[16]));
177 asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
178 "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
179 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
180 "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
181 "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
182 "vpxorq %%zmm4, %%zmm5, %%zmm5"
188 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
189 "vbroadcasti64x2 %1, %%zmm1"
191 : "m" (pbmul[0]), "m" (pbmul[16]));
193 asm volatile("vpsraw $4, %%zmm0, %%zmm2\n\t"
194 "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
195 "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
196 "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
197 "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
198 "vpxorq %%zmm4, %%zmm1, %%zmm1"
203 asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
205 "vmovdqa64 %%zmm1, %0\n\t"
209 asm volatile("vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
210 "vmovdqa64 %%zmm0, %0"
225 static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
229 const u8 *qmul; /* Q multiplier table */
232 p = (u8 *)ptrs[disks-2];
233 q = (u8 *)ptrs[disks-1];
236 * Compute syndrome with zero for the missing data page
237 * Use the dead data page as temporary storage for delta q
240 dq = (u8 *)ptrs[faila];
241 ptrs[faila] = raid6_get_zero_page();
244 raid6_call.gen_syndrome(disks, bytes, ptrs);
246 /* Restore pointer table */
250 /* Now, pick the proper data tables */
251 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
255 asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
259 asm volatile("vmovdqa64 %0, %%zmm3\n\t"
260 "vmovdqa64 %1, %%zmm8\n\t"
261 "vpxorq %2, %%zmm3, %%zmm3\n\t"
262 "vpxorq %3, %%zmm8, %%zmm8"
264 : "m" (dq[0]), "m" (dq[64]), "m" (q[0]),
271 asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
272 "vmovapd %%zmm0, %%zmm13\n\t"
273 "vbroadcasti64x2 %1, %%zmm1\n\t"
274 "vmovapd %%zmm1, %%zmm14"
276 : "m" (qmul[0]), "m" (qmul[16]));
278 asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
279 "vpsraw $4, %%zmm8, %%zmm12\n\t"
280 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
281 "vpandq %%zmm7, %%zmm8, %%zmm8\n\t"
282 "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
283 "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
284 "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
285 "vpshufb %%zmm8, %%zmm13, %%zmm13\n\t"
286 "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
287 "vpshufb %%zmm12, %%zmm14, %%zmm14\n\t"
288 "vpxorq %%zmm0, %%zmm1, %%zmm1\n\t"
289 "vpxorq %%zmm13, %%zmm14, %%zmm14"
294 * 1 = qmul[q[0] ^ dq[0]]
295 * 14 = qmul[q[64] ^ dq[64]]
297 asm volatile("vmovdqa64 %0, %%zmm2\n\t"
298 "vmovdqa64 %1, %%zmm12\n\t"
299 "vpxorq %%zmm1, %%zmm2, %%zmm2\n\t"
300 "vpxorq %%zmm14, %%zmm12, %%zmm12"
302 : "m" (p[0]), "m" (p[64]));
305 * 2 = p[0] ^ qmul[q[0] ^ dq[0]]
306 * 12 = p[64] ^ qmul[q[64] ^ dq[64]]
309 asm volatile("vmovdqa64 %%zmm1, %0\n\t"
310 "vmovdqa64 %%zmm14, %1\n\t"
311 "vmovdqa64 %%zmm2, %2\n\t"
312 "vmovdqa64 %%zmm12,%3"
314 : "m" (dq[0]), "m" (dq[64]), "m" (p[0]),
322 asm volatile("vmovdqa64 %0, %%zmm3\n\t"
323 "vpxorq %1, %%zmm3, %%zmm3"
325 : "m" (dq[0]), "m" (q[0]));
329 asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
330 "vbroadcasti64x2 %1, %%zmm1"
332 : "m" (qmul[0]), "m" (qmul[16]));
334 asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
335 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
336 "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
337 "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
338 "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
339 "vpxorq %%zmm0, %%zmm1, %%zmm1"
343 /* 1 = qmul[q ^ dq] */
345 asm volatile("vmovdqa64 %0, %%zmm2\n\t"
346 "vpxorq %%zmm1, %%zmm2, %%zmm2"
350 /* 2 = p ^ qmul[q ^ dq] */
352 asm volatile("vmovdqa64 %%zmm1, %0\n\t"
353 "vmovdqa64 %%zmm2, %1"
355 : "m" (dq[0]), "m" (p[0]));
367 const struct raid6_recov_calls raid6_recov_avx512 = {
368 .data2 = raid6_2data_recov_avx512,
369 .datap = raid6_datap_recov_avx512,
370 .valid = raid6_has_avx512,