]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/config/rs6000/mma.md
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update copyright years.
[thirdparty/gcc.git] / gcc / config / rs6000 / mma.md
CommitLineData
f002c046 1;; Matrix-Multiply Assist (MMA) patterns.
83ffe9cd 2;; Copyright (C) 2020-2023 Free Software Foundation, Inc.
f002c046
PB		 3;; Contributed by Peter Bergner <bergner@linux.ibm.com> and
4;; Michael Meissner <meissner@linux.ibm.com>
5;;
6;; This file is part of GCC.
7;;
8;; GCC is free software; you can redistribute it and/or modify it
9;; under the terms of the GNU General Public License as published
10;; by the Free Software Foundation; either version 3, or (at your
11;; option) any later version.
12;;
13;; GCC is distributed in the hope that it will be useful, but WITHOUT
14;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
16;; License for more details.
17;;
18;; You should have received a copy of the GNU General Public License
19;; along with GCC; see the file COPYING3. If not see
20;; <http://www.gnu.org/licenses/>.
21
f8f8909a
AS		 22;; The MMA patterns use the multi-register XOmode and OOmode opaque
23;; modes to implement the target specific __vector_quad and
24;; __vector_pair types that the MMA built-in functions reference. We
25;; use OPAQUE_MODE to prevent anything from trying to open them up.
f002c046 26
8ee2640b
PB		 27(define_constants [(MAX_MMA_OPERANDS 7)])
28
29;; Constants for creating unspecs
30
31(define_c_enum "unspec"
15c02ab2 32 [UNSPEC_VSX_ASSEMBLE
f8f8909a 33 UNSPEC_MMA_EXTRACT
8ee2640b
PB		 34 UNSPEC_MMA_PMXVBF16GER2
35 UNSPEC_MMA_PMXVBF16GER2NN
36 UNSPEC_MMA_PMXVBF16GER2NP
37 UNSPEC_MMA_PMXVBF16GER2PN
38 UNSPEC_MMA_PMXVBF16GER2PP
39 UNSPEC_MMA_PMXVF16GER2
40 UNSPEC_MMA_PMXVF16GER2NN
41 UNSPEC_MMA_PMXVF16GER2NP
42 UNSPEC_MMA_PMXVF16GER2PN
43 UNSPEC_MMA_PMXVF16GER2PP
44 UNSPEC_MMA_PMXVF32GER
45 UNSPEC_MMA_PMXVF32GERNN
46 UNSPEC_MMA_PMXVF32GERNP
47 UNSPEC_MMA_PMXVF32GERPN
48 UNSPEC_MMA_PMXVF32GERPP
49 UNSPEC_MMA_PMXVF64GER
50 UNSPEC_MMA_PMXVF64GERNN
51 UNSPEC_MMA_PMXVF64GERNP
52 UNSPEC_MMA_PMXVF64GERPN
53 UNSPEC_MMA_PMXVF64GERPP
54 UNSPEC_MMA_PMXVI16GER2
55 UNSPEC_MMA_PMXVI16GER2PP
56 UNSPEC_MMA_PMXVI16GER2S
57 UNSPEC_MMA_PMXVI16GER2SPP
58 UNSPEC_MMA_PMXVI4GER8
59 UNSPEC_MMA_PMXVI4GER8PP
60 UNSPEC_MMA_PMXVI8GER4
61 UNSPEC_MMA_PMXVI8GER4PP
62 UNSPEC_MMA_PMXVI8GER4SPP
63 UNSPEC_MMA_XVBF16GER2
64 UNSPEC_MMA_XVBF16GER2NN
65 UNSPEC_MMA_XVBF16GER2NP
66 UNSPEC_MMA_XVBF16GER2PN
67 UNSPEC_MMA_XVBF16GER2PP
68 UNSPEC_MMA_XVF16GER2
69 UNSPEC_MMA_XVF16GER2NN
70 UNSPEC_MMA_XVF16GER2NP
71 UNSPEC_MMA_XVF16GER2PN
72 UNSPEC_MMA_XVF16GER2PP
73 UNSPEC_MMA_XVF32GER
74 UNSPEC_MMA_XVF32GERNN
75 UNSPEC_MMA_XVF32GERNP
76 UNSPEC_MMA_XVF32GERPN
77 UNSPEC_MMA_XVF32GERPP
78 UNSPEC_MMA_XVF64GER
79 UNSPEC_MMA_XVF64GERNN
80 UNSPEC_MMA_XVF64GERNP
81 UNSPEC_MMA_XVF64GERPN
82 UNSPEC_MMA_XVF64GERPP
83 UNSPEC_MMA_XVI16GER2
84 UNSPEC_MMA_XVI16GER2PP
85 UNSPEC_MMA_XVI16GER2S
86 UNSPEC_MMA_XVI16GER2SPP
87 UNSPEC_MMA_XVI4GER8
88 UNSPEC_MMA_XVI4GER8PP
89 UNSPEC_MMA_XVI8GER4
90 UNSPEC_MMA_XVI8GER4PP
91 UNSPEC_MMA_XVI8GER4SPP
92 UNSPEC_MMA_XXMFACC
93 UNSPEC_MMA_XXMTACC
f80b9be0
PB		 94 ])
95
96(define_c_enum "unspecv"
15c02ab2
PB		 97 [UNSPECV_MMA_ASSEMBLE
98 UNSPECV_MMA_XXSETACCZ
8ee2640b
PB		 99 ])
100
101;; MMA instructions with 1 accumulator argument
102(define_int_iterator MMA_ACC [UNSPEC_MMA_XXMFACC
103 UNSPEC_MMA_XXMTACC])
104
105;; MMA instructions with 2 vector arguments
106(define_int_iterator MMA_VV [UNSPEC_MMA_XVI4GER8
107 UNSPEC_MMA_XVI8GER4
108 UNSPEC_MMA_XVI16GER2
109 UNSPEC_MMA_XVI16GER2S
110 UNSPEC_MMA_XVF16GER2
111 UNSPEC_MMA_XVBF16GER2
112 UNSPEC_MMA_XVF32GER])
113
114;; MMA instructions with 1 accumulator and 2 vector arguments
115(define_int_iterator MMA_AVV [UNSPEC_MMA_XVI4GER8PP
116 UNSPEC_MMA_XVI8GER4PP
117 UNSPEC_MMA_XVI8GER4SPP
118 UNSPEC_MMA_XVI16GER2PP
119 UNSPEC_MMA_XVI16GER2SPP
120 UNSPEC_MMA_XVF16GER2PP
121 UNSPEC_MMA_XVF16GER2PN
122 UNSPEC_MMA_XVF16GER2NP
123 UNSPEC_MMA_XVF16GER2NN
124 UNSPEC_MMA_XVBF16GER2PP
125 UNSPEC_MMA_XVBF16GER2PN
126 UNSPEC_MMA_XVBF16GER2NP
127 UNSPEC_MMA_XVBF16GER2NN
128 UNSPEC_MMA_XVF32GERPP
129 UNSPEC_MMA_XVF32GERPN
130 UNSPEC_MMA_XVF32GERNP
131 UNSPEC_MMA_XVF32GERNN])
132
133;; MMA instructions with 1 vector pair and 1 vector arguments
134(define_int_iterator MMA_PV [UNSPEC_MMA_XVF64GER])
135
136;; MMA instructions with 1 accumulator, 1 vector pair and 1 vector arguments
137(define_int_iterator MMA_APV [UNSPEC_MMA_XVF64GERPP
138 UNSPEC_MMA_XVF64GERPN
139 UNSPEC_MMA_XVF64GERNP
140 UNSPEC_MMA_XVF64GERNN])
141
142;; MMA instructions with 2 vector, 2 4-bit and 1 8-bit arguments
143(define_int_iterator MMA_VVI4I4I8 [UNSPEC_MMA_PMXVI4GER8])
144
145;; MMA instructions with 1 accumulator, 2 vector, 2 4-bit and 1 8-bit arguments
146(define_int_iterator MMA_AVVI4I4I8 [UNSPEC_MMA_PMXVI4GER8PP])
147
148;; MMA instructions with 2 vector, 2 4-bit and 1 2-bit arguments
149(define_int_iterator MMA_VVI4I4I2 [UNSPEC_MMA_PMXVI16GER2
150 UNSPEC_MMA_PMXVI16GER2S
151 UNSPEC_MMA_PMXVF16GER2
152 UNSPEC_MMA_PMXVBF16GER2])
153
154;; MMA instructions with 1 accumulator, 2 vector, 2 4-bit and 1 2-bit arguments
155(define_int_iterator MMA_AVVI4I4I2 [UNSPEC_MMA_PMXVI16GER2PP
156 UNSPEC_MMA_PMXVI16GER2SPP
157 UNSPEC_MMA_PMXVF16GER2PP
158 UNSPEC_MMA_PMXVF16GER2PN
159 UNSPEC_MMA_PMXVF16GER2NP
160 UNSPEC_MMA_PMXVF16GER2NN
161 UNSPEC_MMA_PMXVBF16GER2PP
162 UNSPEC_MMA_PMXVBF16GER2PN
163 UNSPEC_MMA_PMXVBF16GER2NP
164 UNSPEC_MMA_PMXVBF16GER2NN])
165
166;; MMA instructions with 2 vector and 2 4-bit arguments
167(define_int_iterator MMA_VVI4I4 [UNSPEC_MMA_PMXVF32GER])
168
169;; MMA instructions with 1 accumulator, 2 vector and 2 4-bit arguments
170(define_int_iterator MMA_AVVI4I4 [UNSPEC_MMA_PMXVF32GERPP
171 UNSPEC_MMA_PMXVF32GERPN
172 UNSPEC_MMA_PMXVF32GERNP
173 UNSPEC_MMA_PMXVF32GERNN])
174
175;; MMA instructions with 2 vector, 1 4-bit and 1 2-bit arguments
176(define_int_iterator MMA_PVI4I2 [UNSPEC_MMA_PMXVF64GER])
177
178;; MMA instructions with 1 accumulator, 2 vector, 1 4-bit and 1 2-bit arguments
179(define_int_iterator MMA_APVI4I2 [UNSPEC_MMA_PMXVF64GERPP
180 UNSPEC_MMA_PMXVF64GERPN
181 UNSPEC_MMA_PMXVF64GERNP
182 UNSPEC_MMA_PMXVF64GERNN])
183
184;; MMA instructions with 2 vector and 3 4-bit arguments
185(define_int_iterator MMA_VVI4I4I4 [UNSPEC_MMA_PMXVI8GER4])
186
187;; MMA instructions with 1 accumulator, 2 vector and 3 4-bit arguments
188(define_int_iterator MMA_AVVI4I4I4 [UNSPEC_MMA_PMXVI8GER4PP
189 UNSPEC_MMA_PMXVI8GER4SPP])
190
191(define_int_attr acc [(UNSPEC_MMA_XXMFACC "xxmfacc")
192 (UNSPEC_MMA_XXMTACC "xxmtacc")])
193
194(define_int_attr vv [(UNSPEC_MMA_XVI4GER8 "xvi4ger8")
195 (UNSPEC_MMA_XVI8GER4 "xvi8ger4")
196 (UNSPEC_MMA_XVI16GER2 "xvi16ger2")
197 (UNSPEC_MMA_XVI16GER2S "xvi16ger2s")
198 (UNSPEC_MMA_XVF16GER2 "xvf16ger2")
199 (UNSPEC_MMA_XVBF16GER2 "xvbf16ger2")
200 (UNSPEC_MMA_XVF32GER "xvf32ger")])
201
202(define_int_attr avv [(UNSPEC_MMA_XVI4GER8PP "xvi4ger8pp")
203 (UNSPEC_MMA_XVI8GER4PP "xvi8ger4pp")
204 (UNSPEC_MMA_XVI8GER4SPP "xvi8ger4spp")
205 (UNSPEC_MMA_XVI16GER2PP "xvi16ger2pp")
206 (UNSPEC_MMA_XVI16GER2SPP "xvi16ger2spp")
207 (UNSPEC_MMA_XVF16GER2PP "xvf16ger2pp")
208 (UNSPEC_MMA_XVF16GER2PN "xvf16ger2pn")
209 (UNSPEC_MMA_XVF16GER2NP "xvf16ger2np")
210 (UNSPEC_MMA_XVF16GER2NN "xvf16ger2nn")
211 (UNSPEC_MMA_XVBF16GER2PP "xvbf16ger2pp")
212 (UNSPEC_MMA_XVBF16GER2PN "xvbf16ger2pn")
213 (UNSPEC_MMA_XVBF16GER2NP "xvbf16ger2np")
214 (UNSPEC_MMA_XVBF16GER2NN "xvbf16ger2nn")
215 (UNSPEC_MMA_XVF32GERPP "xvf32gerpp")
216 (UNSPEC_MMA_XVF32GERPN "xvf32gerpn")
217 (UNSPEC_MMA_XVF32GERNP "xvf32gernp")
218 (UNSPEC_MMA_XVF32GERNN "xvf32gernn")])
219
220(define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger")])
221
222(define_int_attr apv [(UNSPEC_MMA_XVF64GERPP "xvf64gerpp")
223 (UNSPEC_MMA_XVF64GERPN "xvf64gerpn")
224 (UNSPEC_MMA_XVF64GERNP "xvf64gernp")
225 (UNSPEC_MMA_XVF64GERNN "xvf64gernn")])
226
227(define_int_attr vvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8 "pmxvi4ger8")])
228
229(define_int_attr avvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8PP "pmxvi4ger8pp")])
230
231(define_int_attr vvi4i4i2 [(UNSPEC_MMA_PMXVI16GER2 "pmxvi16ger2")
232 (UNSPEC_MMA_PMXVI16GER2S "pmxvi16ger2s")
233 (UNSPEC_MMA_PMXVF16GER2 "pmxvf16ger2")
234 (UNSPEC_MMA_PMXVBF16GER2 "pmxvbf16ger2")])
235
236(define_int_attr avvi4i4i2 [(UNSPEC_MMA_PMXVI16GER2PP "pmxvi16ger2pp")
237 (UNSPEC_MMA_PMXVI16GER2SPP "pmxvi16ger2spp")
238 (UNSPEC_MMA_PMXVF16GER2PP "pmxvf16ger2pp")
239 (UNSPEC_MMA_PMXVF16GER2PN "pmxvf16ger2pn")
240 (UNSPEC_MMA_PMXVF16GER2NP "pmxvf16ger2np")
241 (UNSPEC_MMA_PMXVF16GER2NN "pmxvf16ger2nn")
242 (UNSPEC_MMA_PMXVBF16GER2PP "pmxvbf16ger2pp")
243 (UNSPEC_MMA_PMXVBF16GER2PN "pmxvbf16ger2pn")
244 (UNSPEC_MMA_PMXVBF16GER2NP "pmxvbf16ger2np")
245 (UNSPEC_MMA_PMXVBF16GER2NN "pmxvbf16ger2nn")])
246
247(define_int_attr vvi4i4 [(UNSPEC_MMA_PMXVF32GER "pmxvf32ger")])
248
249(define_int_attr avvi4i4 [(UNSPEC_MMA_PMXVF32GERPP "pmxvf32gerpp")
250 (UNSPEC_MMA_PMXVF32GERPN "pmxvf32gerpn")
251 (UNSPEC_MMA_PMXVF32GERNP "pmxvf32gernp")
252 (UNSPEC_MMA_PMXVF32GERNN "pmxvf32gernn")])
253
254(define_int_attr pvi4i2 [(UNSPEC_MMA_PMXVF64GER "pmxvf64ger")])
255
256(define_int_attr apvi4i2 [(UNSPEC_MMA_PMXVF64GERPP "pmxvf64gerpp")
257 (UNSPEC_MMA_PMXVF64GERPN "pmxvf64gerpn")
258 (UNSPEC_MMA_PMXVF64GERNP "pmxvf64gernp")
259 (UNSPEC_MMA_PMXVF64GERNN "pmxvf64gernn")])
260
261(define_int_attr vvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4 "pmxvi8ger4")])
262
263(define_int_attr avvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4PP "pmxvi8ger4pp")
264 (UNSPEC_MMA_PMXVI8GER4SPP "pmxvi8ger4spp")])
265
266
f8f8909a
AS		 267;; Vector pair support. OOmode can only live in VSRs.
268(define_expand "movoo"
269 [(set (match_operand:OO 0 "nonimmediate_operand")
270 (match_operand:OO 1 "input_operand"))]
9367e3a6 271 ""
f002c046 272{
9367e3a6
KL		 273 if (TARGET_MMA)
274 {
275 rs6000_emit_move (operands[0], operands[1], OOmode);
276 DONE;
277 }
278 else if (currently_expanding_to_rtl && seen_error ())
279 {
280 /* PR103353 shows we may want to continue to expand the __builtin_vsx_lxvp
281 built-in function, even if we have already emitted error messages about
282 some missing required conditions. As shown in that PR, without one
283 explicit mov optab on OOmode provided, it would call emit_move_insn
284 recursively. So we allow this pattern to be generated when we are
285 expanding to RTL and have seen errors. It would not cause further ICEs
286 as the compilation would stop soon after expanding. */
287 }
282462b3
KL		 288 else if (rs6000_opaque_type_invalid_use_p (currently_expanding_gimple_stmt))
289 ;
9367e3a6 290 else
282462b3
KL		 291 /* Catch unexpected cases. */
292 gcc_assert (false);
f002c046
PB		 293})
294
f8f8909a
AS		 295(define_insn_and_split "*movoo"
296 [(set (match_operand:OO 0 "nonimmediate_operand" "=wa,m,wa")
297 (match_operand:OO 1 "input_operand" "m,wa,wa"))]
f002c046 298 "TARGET_MMA
f8f8909a
AS		 299 && (gpc_reg_operand (operands[0], OOmode)
300 || gpc_reg_operand (operands[1], OOmode))"
f002c046
PB		 301 "@
302 lxvp%X1 %x0,%1
303 stxvp%X0 %x1,%0
304 #"
305 "&& reload_completed
306 && (!MEM_P (operands[0]) && !MEM_P (operands[1]))"
307 [(const_int 0)]
308{
309 rs6000_split_multireg_move (operands[0], operands[1]);
310 DONE;
311}
312 [(set_attr "type" "vecload,vecstore,veclogical")
9a5a1e27 313 (set_attr "size" "256")
f002c046
PB		 314 (set_attr "length" "*,*,8")])
315
316\f
f8f8909a
AS		 317;; Vector quad support. XOmode can only live in FPRs.
318(define_expand "movxo"
319 [(set (match_operand:XO 0 "nonimmediate_operand")
320 (match_operand:XO 1 "input_operand"))]
9367e3a6 321 ""
f002c046 322{
9367e3a6
KL		 323 if (TARGET_MMA)
324 {
325 rs6000_emit_move (operands[0], operands[1], XOmode);
326 DONE;
327 }
328 else if (currently_expanding_to_rtl && seen_error ())
329 {
330 /* PR103353 shows we may want to continue to expand the __builtin_vsx_lxvp
331 built-in function, even if we have already emitted error messages about
332 some missing required conditions. So do the same handlings for XOmode
333 as OOmode here. */
334 }
282462b3
KL		 335 else if (rs6000_opaque_type_invalid_use_p (currently_expanding_gimple_stmt))
336 ;
9367e3a6 337 else
282462b3
KL		 338 /* Catch unexpected cases. */
339 gcc_assert (false);
f002c046
PB		 340})
341
f8f8909a
AS		 342(define_insn_and_split "*movxo"
343 [(set (match_operand:XO 0 "nonimmediate_operand" "=d,m,d")
344 (match_operand:XO 1 "input_operand" "m,d,d"))]
f002c046 345 "TARGET_MMA
f8f8909a
AS		 346 && (gpc_reg_operand (operands[0], XOmode)
347 || gpc_reg_operand (operands[1], XOmode))"
9c376d1c
PB		 348 "@
349 #
350 #
f8f8909a
AS		 351 #"
352 "&& reload_completed"
f002c046
PB		 353 [(const_int 0)]
354{
355 rs6000_split_multireg_move (operands[0], operands[1]);
356 DONE;
357}
f8f8909a 358 [(set_attr "type" "vecload,vecstore,veclogical")
e128aaa6 359 (set_attr "length" "*,*,16")
f8f8909a 360 (set_attr "max_prefixed_insns" "2,2,*")])
8ee2640b 361
77ef995c 362(define_expand "vsx_assemble_pair"
f8f8909a
AS		 363 [(match_operand:OO 0 "vsx_register_operand")
364 (match_operand:V16QI 1 "mma_assemble_input_operand")
365 (match_operand:V16QI 2 "mma_assemble_input_operand")]
8ee2640b
PB		 366 "TARGET_MMA"
367{
f8f8909a
AS		 368 rtx src = gen_rtx_UNSPEC (OOmode,
369 gen_rtvec (2, operands[1], operands[2]),
15c02ab2 370 UNSPEC_VSX_ASSEMBLE);
f8f8909a
AS		 371 emit_move_insn (operands[0], src);
372 DONE;
373})
8ee2640b 374
4cdf7db9
PB		 375;; We cannot update the two output registers atomically, so mark the output
376;; as an early clobber so we don't accidentally clobber the input operands. */
377
77ef995c 378(define_insn_and_split "*vsx_assemble_pair"
4cdf7db9 379 [(set (match_operand:OO 0 "vsx_register_operand" "=&wa")
f8f8909a
AS		 380 (unspec:OO [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa")
381 (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa")]
15c02ab2 382 UNSPEC_VSX_ASSEMBLE))]
f8f8909a
AS		 383 "TARGET_MMA"
384 "#"
385 "&& reload_completed"
386 [(const_int 0)]
387{
388 rtx src = gen_rtx_UNSPEC (OOmode,
389 gen_rtvec (2, operands[1], operands[2]),
15c02ab2 390 UNSPEC_VSX_ASSEMBLE);
f8f8909a
AS		 391 rs6000_split_multireg_move (operands[0], src);
392 DONE;
393})
394
77ef995c 395(define_expand "vsx_disassemble_pair"
f8f8909a
AS		 396 [(match_operand:V16QI 0 "mma_disassemble_output_operand")
397 (match_operand:OO 1 "vsx_register_operand")
398 (match_operand 2 "const_0_to_1_operand")]
399 "TARGET_MMA"
400{
401 rtx src;
402 int regoff = INTVAL (operands[2]);
403 src = gen_rtx_UNSPEC (V16QImode,
404 gen_rtvec (2, operands[1], GEN_INT (regoff)),
405 UNSPEC_MMA_EXTRACT);
406 emit_move_insn (operands[0], src);
407 DONE;
408})
8ee2640b 409
77ef995c 410(define_insn_and_split "*vsx_disassemble_pair"
f8f8909a
AS		 411 [(set (match_operand:V16QI 0 "mma_disassemble_output_operand" "=mwa")
412 (unspec:V16QI [(match_operand:OO 1 "vsx_register_operand" "wa")
413 (match_operand 2 "const_0_to_1_operand")]
414 UNSPEC_MMA_EXTRACT))]
415 "TARGET_MMA
416 && vsx_register_operand (operands[1], OOmode)"
417 "#"
418 "&& reload_completed"
419 [(const_int 0)]
420{
421 int reg = REGNO (operands[1]);
422 int regoff = INTVAL (operands[2]);
423 rtx src = gen_rtx_REG (V16QImode, reg + regoff);
424 emit_move_insn (operands[0], src);
8ee2640b
PB		 425 DONE;
426})
427
428(define_expand "mma_assemble_acc"
f8f8909a
AS		 429 [(match_operand:XO 0 "fpr_reg_operand")
430 (match_operand:V16QI 1 "mma_assemble_input_operand")
431 (match_operand:V16QI 2 "mma_assemble_input_operand")
432 (match_operand:V16QI 3 "mma_assemble_input_operand")
433 (match_operand:V16QI 4 "mma_assemble_input_operand")]
8ee2640b
PB		 434 "TARGET_MMA"
435{
15c02ab2
PB		 436 rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode,
437 gen_rtvec (4, operands[1], operands[2],
438 operands[3], operands[4]),
439 UNSPECV_MMA_ASSEMBLE);
8ee2640b
PB		 440 emit_move_insn (operands[0], src);
441 DONE;
442})
443
4cdf7db9
PB		 444;; We cannot update the four output registers atomically, so mark the output
445;; as an early clobber so we don't accidentally clobber the input operands. */
446
8ee2640b 447(define_insn_and_split "*mma_assemble_acc"
4cdf7db9 448 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d")
15c02ab2
PB		 449 (unspec_volatile:XO
450 [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa")
451 (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa")
452 (match_operand:V16QI 3 "mma_assemble_input_operand" "mwa")
453 (match_operand:V16QI 4 "mma_assemble_input_operand" "mwa")]
454 UNSPECV_MMA_ASSEMBLE))]
8ee2640b 455 "TARGET_MMA
f8f8909a 456 && fpr_reg_operand (operands[0], XOmode)"
8ee2640b
PB		 457 "#"
458 "&& reload_completed"
459 [(const_int 0)]
460{
15c02ab2
PB		 461 rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode,
462 gen_rtvec (4, operands[1], operands[2],
463 operands[3], operands[4]),
464 UNSPECV_MMA_ASSEMBLE);
8ee2640b
PB		 465 rs6000_split_multireg_move (operands[0], src);
466 DONE;
467})
468
f8f8909a
AS		 469(define_expand "mma_disassemble_acc"
470 [(match_operand:V16QI 0 "mma_disassemble_output_operand")
471 (match_operand:XO 1 "fpr_reg_operand")
472 (match_operand 2 "const_0_to_3_operand")]
473 "TARGET_MMA"
474{
475 rtx src;
476 int regoff = INTVAL (operands[2]);
477 src = gen_rtx_UNSPEC (V16QImode,
478 gen_rtvec (2, operands[1], GEN_INT (regoff)),
479 UNSPEC_MMA_EXTRACT);
480 emit_move_insn (operands[0], src);
481 DONE;
482})
483
484(define_insn_and_split "*mma_disassemble_acc"
485 [(set (match_operand:V16QI 0 "mma_disassemble_output_operand" "=mwa")
486 (unspec:V16QI [(match_operand:XO 1 "fpr_reg_operand" "d")
487 (match_operand 2 "const_0_to_3_operand")]
488 UNSPEC_MMA_EXTRACT))]
489 "TARGET_MMA
490 && fpr_reg_operand (operands[1], XOmode)"
491 "#"
492 "&& reload_completed"
493 [(const_int 0)]
494{
495 int reg = REGNO (operands[1]);
496 int regoff = INTVAL (operands[2]);
497 rtx src = gen_rtx_REG (V16QImode, reg + regoff);
498 emit_move_insn (operands[0], src);
499 DONE;
500})
501
8ee2640b
PB		 502;; MMA instructions that do not use their accumulators as an input, still
503;; must not allow their vector operands to overlap the registers used by
504;; the accumulator. We enforce this by marking the output as early clobber.
505
506(define_insn "mma_<acc>"
f8f8909a
AS		 507 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d")
508 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0")]
8ee2640b
PB		 509 MMA_ACC))]
510 "TARGET_MMA"
511 "<acc> %A0"
512 [(set_attr "type" "mma")])
513
f80b9be0
PB		 514;; We can't have integer constants in XOmode so we wrap this in an
515;; UNSPEC_VOLATILE.
f8f8909a 516
f80b9be0 517(define_insn "mma_xxsetaccz"
f8f8909a 518 [(set (match_operand:XO 0 "fpr_reg_operand" "=d")
f80b9be0
PB		 519 (unspec_volatile:XO [(const_int 0)]
520 UNSPECV_MMA_XXSETACCZ))]
f8f8909a
AS		 521 "TARGET_MMA"
522 "xxsetaccz %A0"
f80b9be0 523 [(set_attr "type" "mma")])
f8f8909a 524
8ee2640b 525(define_insn "mma_<vv>"
c6e36f05
PB		 526 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
527 (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
528 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")]
f8f8909a 529 MMA_VV))]
8ee2640b
PB		 530 "TARGET_MMA"
531 "<vv> %A0,%x1,%x2"
532 [(set_attr "type" "mma")])
533
534(define_insn "mma_<avv>"
c6e36f05
PB		 535 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
536 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
537 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
538 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")]
f8f8909a 539 MMA_AVV))]
8ee2640b
PB		 540 "TARGET_MMA"
541 "<avv> %A0,%x2,%x3"
542 [(set_attr "type" "mma")])
543
544(define_insn "mma_<pv>"
c6e36f05
PB		 545 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
546 (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "v,?wa")
547 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")]
f8f8909a 548 MMA_PV))]
8ee2640b
PB		 549 "TARGET_MMA"
550 "<pv> %A0,%x1,%x2"
551 [(set_attr "type" "mma")])
552
553(define_insn "mma_<apv>"
c6e36f05
PB		 554 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
555 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
556 (match_operand:OO 2 "vsx_register_operand" "v,?wa")
557 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")]
f8f8909a 558 MMA_APV))]
8ee2640b
PB		 559 "TARGET_MMA"
560 "<apv> %A0,%x2,%x3"
561 [(set_attr "type" "mma")])
562
563(define_insn "mma_<vvi4i4i8>"
c6e36f05
PB		 564 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
565 (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
566 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
567 (match_operand:SI 3 "const_0_to_15_operand" "n,n")
568 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
569 (match_operand:SI 5 "u8bit_cint_operand" "n,n")]
f8f8909a 570 MMA_VVI4I4I8))]
8ee2640b
PB		 571 "TARGET_MMA"
572 "<vvi4i4i8> %A0,%x1,%x2,%3,%4,%5"
ea9a39e6
PH		 573 [(set_attr "type" "mma")
574 (set_attr "prefixed" "yes")])
8ee2640b
PB		 575
576(define_insn "mma_<avvi4i4i8>"
c6e36f05
PB		 577 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
578 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
579 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
580 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
581 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
582 (match_operand:SI 5 "const_0_to_15_operand" "n,n")
583 (match_operand:SI 6 "u8bit_cint_operand" "n,n")]
f8f8909a 584 MMA_AVVI4I4I8))]
8ee2640b
PB		 585 "TARGET_MMA"
586 "<avvi4i4i8> %A0,%x2,%x3,%4,%5,%6"
ea9a39e6
PH		 587 [(set_attr "type" "mma")
588 (set_attr "prefixed" "yes")])
8ee2640b
PB		 589
590(define_insn "mma_<vvi4i4i2>"
c6e36f05
PB		 591 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
592 (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
593 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
594 (match_operand:SI 3 "const_0_to_15_operand" "n,n")
595 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
596 (match_operand:SI 5 "const_0_to_3_operand" "n,n")]
f8f8909a 597 MMA_VVI4I4I2))]
8ee2640b
PB		 598 "TARGET_MMA"
599 "<vvi4i4i2> %A0,%x1,%x2,%3,%4,%5"
ea9a39e6
PH		 600 [(set_attr "type" "mma")
601 (set_attr "prefixed" "yes")])
8ee2640b
PB		 602
603(define_insn "mma_<avvi4i4i2>"
c6e36f05
PB		 604 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
605 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
606 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
607 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
608 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
609 (match_operand:SI 5 "const_0_to_15_operand" "n,n")
610 (match_operand:SI 6 "const_0_to_3_operand" "n,n")]
f8f8909a 611 MMA_AVVI4I4I2))]
8ee2640b
PB		 612 "TARGET_MMA"
613 "<avvi4i4i2> %A0,%x2,%x3,%4,%5,%6"
ea9a39e6
PH		 614 [(set_attr "type" "mma")
615 (set_attr "prefixed" "yes")])
8ee2640b
PB		 616
617(define_insn "mma_<vvi4i4>"
c6e36f05
PB		 618 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
619 (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
620 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
621 (match_operand:SI 3 "const_0_to_15_operand" "n,n")
622 (match_operand:SI 4 "const_0_to_15_operand" "n,n")]
f8f8909a 623 MMA_VVI4I4))]
8ee2640b
PB		 624 "TARGET_MMA"
625 "<vvi4i4> %A0,%x1,%x2,%3,%4"
ea9a39e6
PH		 626 [(set_attr "type" "mma")
627 (set_attr "prefixed" "yes")])
8ee2640b
PB		 628
629(define_insn "mma_<avvi4i4>"
c6e36f05
PB		 630 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
631 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
632 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
633 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
634 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
635 (match_operand:SI 5 "const_0_to_15_operand" "n,n")]
f8f8909a 636 MMA_AVVI4I4))]
8ee2640b
PB		 637 "TARGET_MMA"
638 "<avvi4i4> %A0,%x2,%x3,%4,%5"
ea9a39e6
PH		 639 [(set_attr "type" "mma")
640 (set_attr "prefixed" "yes")])
8ee2640b
PB		 641
642(define_insn "mma_<pvi4i2>"
c6e36f05
PB		 643 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
644 (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "v,?wa")
645 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
646 (match_operand:SI 3 "const_0_to_15_operand" "n,n")
647 (match_operand:SI 4 "const_0_to_3_operand" "n,n")]
f8f8909a 648 MMA_PVI4I2))]
8ee2640b
PB		 649 "TARGET_MMA"
650 "<pvi4i2> %A0,%x1,%x2,%3,%4"
ea9a39e6
PH		 651 [(set_attr "type" "mma")
652 (set_attr "prefixed" "yes")])
8ee2640b
PB		 653
654(define_insn "mma_<apvi4i2>"
c6e36f05
PB		 655 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
656 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
657 (match_operand:OO 2 "vsx_register_operand" "v,?wa")
658 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
659 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
660 (match_operand:SI 5 "const_0_to_3_operand" "n,n")]
f8f8909a 661 MMA_APVI4I2))]
8ee2640b
PB		 662 "TARGET_MMA"
663 "<apvi4i2> %A0,%x2,%x3,%4,%5"
ea9a39e6
PH		 664 [(set_attr "type" "mma")
665 (set_attr "prefixed" "yes")])
8ee2640b
PB		 666
667(define_insn "mma_<vvi4i4i4>"
c6e36f05
PB		 668 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
669 (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
670 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
671 (match_operand:SI 3 "const_0_to_15_operand" "n,n")
672 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
673 (match_operand:SI 5 "const_0_to_15_operand" "n,n")]
f8f8909a 674 MMA_VVI4I4I4))]
8ee2640b
PB		 675 "TARGET_MMA"
676 "<vvi4i4i4> %A0,%x1,%x2,%3,%4,%5"
ea9a39e6
PH		 677 [(set_attr "type" "mma")
678 (set_attr "prefixed" "yes")])
8ee2640b
PB		 679
680(define_insn "mma_<avvi4i4i4>"
c6e36f05
PB		 681 [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
682 (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
683 (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
684 (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
685 (match_operand:SI 4 "const_0_to_15_operand" "n,n")
686 (match_operand:SI 5 "const_0_to_15_operand" "n,n")
687 (match_operand:SI 6 "const_0_to_15_operand" "n,n")]
f8f8909a 688 MMA_AVVI4I4I4))]
8ee2640b
PB		 689 "TARGET_MMA"
690 "<avvi4i4i4> %A0,%x2,%x3,%4,%5,%6"
ea9a39e6
PH		 691 [(set_attr "type" "mma")
692 (set_attr "prefixed" "yes")])
Mirror of https://gcc.gnu.org/git/gcc.git