]> git.ipfire.org Git - thirdparty/gcc.git/blob - gcc/config/aarch64/aarch64-simd.md
projects / thirdparty / gcc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
gcc/ada/
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64-simd.md
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2014 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
20
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
23 (match_operand:VALL 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
30
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
43
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r, w")))]
47 "TARGET_SIMD"
48 "@
49 dup\\t%0.<Vtype>, %<vw>1
50 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
51 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
52 )
53
54 (define_insn "aarch64_simd_dup<mode>"
55 [(set (match_operand:VDQF 0 "register_operand" "=w")
56 (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
57 "TARGET_SIMD"
58 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
59 [(set_attr "type" "neon_dup<q>")]
60 )
61
62 (define_insn "aarch64_dup_lane<mode>"
63 [(set (match_operand:VALL 0 "register_operand" "=w")
64 (vec_duplicate:VALL
65 (vec_select:<VEL>
66 (match_operand:VALL 1 "register_operand" "w")
67 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
68 )))]
69 "TARGET_SIMD"
70 {
71 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
72 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
73 }
74 [(set_attr "type" "neon_dup<q>")]
75 )
76
77 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
78 [(set (match_operand:VALL 0 "register_operand" "=w")
79 (vec_duplicate:VALL
80 (vec_select:<VEL>
81 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
82 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
83 )))]
84 "TARGET_SIMD"
85 {
86 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
87 INTVAL (operands[2])));
88 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
89 }
90 [(set_attr "type" "neon_dup<q>")]
91 )
92
93 (define_insn "*aarch64_simd_mov<mode>"
94 [(set (match_operand:VD 0 "nonimmediate_operand"
95 "=w, m, w, ?r, ?w, ?r, w")
96 (match_operand:VD 1 "general_operand"
97 "m, w, w, w, r, r, Dn"))]
98 "TARGET_SIMD
99 && (register_operand (operands[0], <MODE>mode)
100 || register_operand (operands[1], <MODE>mode))"
101 {
102 switch (which_alternative)
103 {
104 case 0: return "ldr\\t%d0, %1";
105 case 1: return "str\\t%d1, %0";
106 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
107 case 3: return "umov\t%0, %1.d[0]";
108 case 4: return "ins\t%0.d[0], %1";
109 case 5: return "mov\t%0, %1";
110 case 6:
111 return aarch64_output_simd_mov_immediate (operands[1],
112 <MODE>mode, 64);
113 default: gcc_unreachable ();
114 }
115 }
116 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
117 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
118 mov_reg, neon_move<q>")]
119 )
120
121 (define_insn "*aarch64_simd_mov<mode>"
122 [(set (match_operand:VQ 0 "nonimmediate_operand"
123 "=w, m, w, ?r, ?w, ?r, w")
124 (match_operand:VQ 1 "general_operand"
125 "m, w, w, w, r, r, Dn"))]
126 "TARGET_SIMD
127 && (register_operand (operands[0], <MODE>mode)
128 || register_operand (operands[1], <MODE>mode))"
129 {
130 switch (which_alternative)
131 {
132 case 0:
133 return "ldr\\t%q0, %1";
134 case 1:
135 return "str\\t%q1, %0";
136 case 2:
137 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
138 case 3:
139 case 4:
140 case 5:
141 return "#";
142 case 6:
143 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
144 default:
145 gcc_unreachable ();
146 }
147 }
148 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
149 neon_logic<q>, multiple, multiple, multiple,\
150 neon_move<q>")
151 (set_attr "length" "4,4,4,8,8,8,4")]
152 )
153
154 (define_split
155 [(set (match_operand:VQ 0 "register_operand" "")
156 (match_operand:VQ 1 "register_operand" ""))]
157 "TARGET_SIMD && reload_completed
158 && GP_REGNUM_P (REGNO (operands[0]))
159 && GP_REGNUM_P (REGNO (operands[1]))"
160 [(set (match_dup 0) (match_dup 1))
161 (set (match_dup 2) (match_dup 3))]
162 {
163 int rdest = REGNO (operands[0]);
164 int rsrc = REGNO (operands[1]);
165 rtx dest[2], src[2];
166
167 dest[0] = gen_rtx_REG (DImode, rdest);
168 src[0] = gen_rtx_REG (DImode, rsrc);
169 dest[1] = gen_rtx_REG (DImode, rdest + 1);
170 src[1] = gen_rtx_REG (DImode, rsrc + 1);
171
172 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
173 })
174
175 (define_split
176 [(set (match_operand:VQ 0 "register_operand" "")
177 (match_operand:VQ 1 "register_operand" ""))]
178 "TARGET_SIMD && reload_completed
179 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
180 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
181 [(const_int 0)]
182 {
183 aarch64_split_simd_move (operands[0], operands[1]);
184 DONE;
185 })
186
187 (define_expand "aarch64_split_simd_mov<mode>"
188 [(set (match_operand:VQ 0)
189 (match_operand:VQ 1))]
190 "TARGET_SIMD"
191 {
192 rtx dst = operands[0];
193 rtx src = operands[1];
194
195 if (GP_REGNUM_P (REGNO (src)))
196 {
197 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
198 rtx src_high_part = gen_highpart (<VHALF>mode, src);
199
200 emit_insn
201 (gen_move_lo_quad_<mode> (dst, src_low_part));
202 emit_insn
203 (gen_move_hi_quad_<mode> (dst, src_high_part));
204 }
205
206 else
207 {
208 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
209 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
210 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
211 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
212
213 emit_insn
214 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
215 emit_insn
216 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
217 }
218 DONE;
219 }
220 )
221
222 (define_insn "aarch64_simd_mov_from_<mode>low"
223 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
224 (vec_select:<VHALF>
225 (match_operand:VQ 1 "register_operand" "w")
226 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
227 "TARGET_SIMD && reload_completed"
228 "umov\t%0, %1.d[0]"
229 [(set_attr "type" "neon_to_gp<q>")
230 (set_attr "length" "4")
231 ])
232
233 (define_insn "aarch64_simd_mov_from_<mode>high"
234 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
235 (vec_select:<VHALF>
236 (match_operand:VQ 1 "register_operand" "w")
237 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
238 "TARGET_SIMD && reload_completed"
239 "umov\t%0, %1.d[1]"
240 [(set_attr "type" "neon_to_gp<q>")
241 (set_attr "length" "4")
242 ])
243
244 (define_insn "orn<mode>3"
245 [(set (match_operand:VDQ 0 "register_operand" "=w")
246 (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
247 (match_operand:VDQ 2 "register_operand" "w")))]
248 "TARGET_SIMD"
249 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
250 [(set_attr "type" "neon_logic<q>")]
251 )
252
253 (define_insn "bic<mode>3"
254 [(set (match_operand:VDQ 0 "register_operand" "=w")
255 (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
256 (match_operand:VDQ 2 "register_operand" "w")))]
257 "TARGET_SIMD"
258 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
259 [(set_attr "type" "neon_logic<q>")]
260 )
261
262 (define_insn "add<mode>3"
263 [(set (match_operand:VDQ 0 "register_operand" "=w")
264 (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
265 (match_operand:VDQ 2 "register_operand" "w")))]
266 "TARGET_SIMD"
267 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
268 [(set_attr "type" "neon_add<q>")]
269 )
270
271 (define_insn "sub<mode>3"
272 [(set (match_operand:VDQ 0 "register_operand" "=w")
273 (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
274 (match_operand:VDQ 2 "register_operand" "w")))]
275 "TARGET_SIMD"
276 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
277 [(set_attr "type" "neon_sub<q>")]
278 )
279
280 (define_insn "mul<mode>3"
281 [(set (match_operand:VDQM 0 "register_operand" "=w")
282 (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
283 (match_operand:VDQM 2 "register_operand" "w")))]
284 "TARGET_SIMD"
285 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
286 [(set_attr "type" "neon_mul_<Vetype><q>")]
287 )
288
289 (define_insn "bswap<mode>"
290 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
291 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
292 "TARGET_SIMD"
293 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
294 [(set_attr "type" "neon_rev<q>")]
295 )
296
297 (define_insn "aarch64_rbit<mode>"
298 [(set (match_operand:VB 0 "register_operand" "=w")
299 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
300 UNSPEC_RBIT))]
301 "TARGET_SIMD"
302 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
303 [(set_attr "type" "neon_rbit")]
304 )
305
306 (define_insn "*aarch64_mul3_elt<mode>"
307 [(set (match_operand:VMUL 0 "register_operand" "=w")
308 (mult:VMUL
309 (vec_duplicate:VMUL
310 (vec_select:<VEL>
311 (match_operand:VMUL 1 "register_operand" "<h_con>")
312 (parallel [(match_operand:SI 2 "immediate_operand")])))
313 (match_operand:VMUL 3 "register_operand" "w")))]
314 "TARGET_SIMD"
315 {
316 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
317 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
318 }
319 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
320 )
321
322 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
323 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
324 (mult:VMUL_CHANGE_NLANES
325 (vec_duplicate:VMUL_CHANGE_NLANES
326 (vec_select:<VEL>
327 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
328 (parallel [(match_operand:SI 2 "immediate_operand")])))
329 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
330 "TARGET_SIMD"
331 {
332 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
333 INTVAL (operands[2])));
334 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
335 }
336 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
337 )
338
339 (define_insn "*aarch64_mul3_elt_to_128df"
340 [(set (match_operand:V2DF 0 "register_operand" "=w")
341 (mult:V2DF
342 (vec_duplicate:V2DF
343 (match_operand:DF 2 "register_operand" "w"))
344 (match_operand:V2DF 1 "register_operand" "w")))]
345 "TARGET_SIMD"
346 "fmul\\t%0.2d, %1.2d, %2.d[0]"
347 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
348 )
349
350 (define_insn "*aarch64_mul3_elt_to_64v2df"
351 [(set (match_operand:DF 0 "register_operand" "=w")
352 (mult:DF
353 (vec_select:DF
354 (match_operand:V2DF 1 "register_operand" "w")
355 (parallel [(match_operand:SI 2 "immediate_operand")]))
356 (match_operand:DF 3 "register_operand" "w")))]
357 "TARGET_SIMD"
358 {
359 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
360 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
361 }
362 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
363 )
364
365 (define_insn "neg<mode>2"
366 [(set (match_operand:VDQ 0 "register_operand" "=w")
367 (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
368 "TARGET_SIMD"
369 "neg\t%0.<Vtype>, %1.<Vtype>"
370 [(set_attr "type" "neon_neg<q>")]
371 )
372
373 (define_insn "abs<mode>2"
374 [(set (match_operand:VDQ 0 "register_operand" "=w")
375 (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
376 "TARGET_SIMD"
377 "abs\t%0.<Vtype>, %1.<Vtype>"
378 [(set_attr "type" "neon_abs<q>")]
379 )
380
381 (define_insn "abd<mode>_3"
382 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
383 (abs:VDQ_BHSI (minus:VDQ_BHSI
384 (match_operand:VDQ_BHSI 1 "register_operand" "w")
385 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
386 "TARGET_SIMD"
387 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
388 [(set_attr "type" "neon_abd<q>")]
389 )
390
391 (define_insn "aba<mode>_3"
392 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
393 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
394 (match_operand:VDQ_BHSI 1 "register_operand" "w")
395 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
396 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
397 "TARGET_SIMD"
398 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
399 [(set_attr "type" "neon_arith_acc<q>")]
400 )
401
402 (define_insn "fabd<mode>_3"
403 [(set (match_operand:VDQF 0 "register_operand" "=w")
404 (abs:VDQF (minus:VDQF
405 (match_operand:VDQF 1 "register_operand" "w")
406 (match_operand:VDQF 2 "register_operand" "w"))))]
407 "TARGET_SIMD"
408 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
409 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
410 )
411
412 (define_insn "*fabd_scalar<mode>3"
413 [(set (match_operand:GPF 0 "register_operand" "=w")
414 (abs:GPF (minus:GPF
415 (match_operand:GPF 1 "register_operand" "w")
416 (match_operand:GPF 2 "register_operand" "w"))))]
417 "TARGET_SIMD"
418 "fabd\t%<s>0, %<s>1, %<s>2"
419 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
420 )
421
422 (define_insn "and<mode>3"
423 [(set (match_operand:VDQ 0 "register_operand" "=w")
424 (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
425 (match_operand:VDQ 2 "register_operand" "w")))]
426 "TARGET_SIMD"
427 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
428 [(set_attr "type" "neon_logic<q>")]
429 )
430
431 (define_insn "ior<mode>3"
432 [(set (match_operand:VDQ 0 "register_operand" "=w")
433 (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
434 (match_operand:VDQ 2 "register_operand" "w")))]
435 "TARGET_SIMD"
436 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
437 [(set_attr "type" "neon_logic<q>")]
438 )
439
440 (define_insn "xor<mode>3"
441 [(set (match_operand:VDQ 0 "register_operand" "=w")
442 (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
443 (match_operand:VDQ 2 "register_operand" "w")))]
444 "TARGET_SIMD"
445 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
446 [(set_attr "type" "neon_logic<q>")]
447 )
448
449 (define_insn "one_cmpl<mode>2"
450 [(set (match_operand:VDQ 0 "register_operand" "=w")
451 (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
452 "TARGET_SIMD"
453 "not\t%0.<Vbtype>, %1.<Vbtype>"
454 [(set_attr "type" "neon_logic<q>")]
455 )
456
457 (define_insn "aarch64_simd_vec_set<mode>"
458 [(set (match_operand:VQ_S 0 "register_operand" "=w,w")
459 (vec_merge:VQ_S
460 (vec_duplicate:VQ_S
461 (match_operand:<VEL> 1 "register_operand" "r,w"))
462 (match_operand:VQ_S 3 "register_operand" "0,0")
463 (match_operand:SI 2 "immediate_operand" "i,i")))]
464 "TARGET_SIMD"
465 {
466 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
467 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
468 switch (which_alternative)
469 {
470 case 0:
471 return "ins\\t%0.<Vetype>[%p2], %w1";
472 case 1:
473 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
474 default:
475 gcc_unreachable ();
476 }
477 }
478 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>")]
479 )
480
481 (define_insn "aarch64_simd_lshr<mode>"
482 [(set (match_operand:VDQ 0 "register_operand" "=w")
483 (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
484 (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))]
485 "TARGET_SIMD"
486 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
487 [(set_attr "type" "neon_shift_imm<q>")]
488 )
489
490 (define_insn "aarch64_simd_ashr<mode>"
491 [(set (match_operand:VDQ 0 "register_operand" "=w")
492 (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
493 (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))]
494 "TARGET_SIMD"
495 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
496 [(set_attr "type" "neon_shift_imm<q>")]
497 )
498
499 (define_insn "aarch64_simd_imm_shl<mode>"
500 [(set (match_operand:VDQ 0 "register_operand" "=w")
501 (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
502 (match_operand:VDQ 2 "aarch64_simd_lshift_imm" "Dl")))]
503 "TARGET_SIMD"
504 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
505 [(set_attr "type" "neon_shift_imm<q>")]
506 )
507
508 (define_insn "aarch64_simd_reg_sshl<mode>"
509 [(set (match_operand:VDQ 0 "register_operand" "=w")
510 (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
511 (match_operand:VDQ 2 "register_operand" "w")))]
512 "TARGET_SIMD"
513 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
514 [(set_attr "type" "neon_shift_reg<q>")]
515 )
516
517 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
518 [(set (match_operand:VDQ 0 "register_operand" "=w")
519 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
520 (match_operand:VDQ 2 "register_operand" "w")]
521 UNSPEC_ASHIFT_UNSIGNED))]
522 "TARGET_SIMD"
523 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
524 [(set_attr "type" "neon_shift_reg<q>")]
525 )
526
527 (define_insn "aarch64_simd_reg_shl<mode>_signed"
528 [(set (match_operand:VDQ 0 "register_operand" "=w")
529 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
530 (match_operand:VDQ 2 "register_operand" "w")]
531 UNSPEC_ASHIFT_SIGNED))]
532 "TARGET_SIMD"
533 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
534 [(set_attr "type" "neon_shift_reg<q>")]
535 )
536
537 (define_expand "ashl<mode>3"
538 [(match_operand:VDQ 0 "register_operand" "")
539 (match_operand:VDQ 1 "register_operand" "")
540 (match_operand:SI 2 "general_operand" "")]
541 "TARGET_SIMD"
542 {
543 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
544 int shift_amount;
545
546 if (CONST_INT_P (operands[2]))
547 {
548 shift_amount = INTVAL (operands[2]);
549 if (shift_amount >= 0 && shift_amount < bit_width)
550 {
551 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
552 shift_amount);
553 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
554 operands[1],
555 tmp));
556 DONE;
557 }
558 else
559 {
560 operands[2] = force_reg (SImode, operands[2]);
561 }
562 }
563 else if (MEM_P (operands[2]))
564 {
565 operands[2] = force_reg (SImode, operands[2]);
566 }
567
568 if (REG_P (operands[2]))
569 {
570 rtx tmp = gen_reg_rtx (<MODE>mode);
571 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
572 convert_to_mode (<VEL>mode,
573 operands[2],
574 0)));
575 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
576 tmp));
577 DONE;
578 }
579 else
580 FAIL;
581 }
582 )
583
584 (define_expand "lshr<mode>3"
585 [(match_operand:VDQ 0 "register_operand" "")
586 (match_operand:VDQ 1 "register_operand" "")
587 (match_operand:SI 2 "general_operand" "")]
588 "TARGET_SIMD"
589 {
590 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
591 int shift_amount;
592
593 if (CONST_INT_P (operands[2]))
594 {
595 shift_amount = INTVAL (operands[2]);
596 if (shift_amount > 0 && shift_amount <= bit_width)
597 {
598 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
599 shift_amount);
600 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
601 operands[1],
602 tmp));
603 DONE;
604 }
605 else
606 operands[2] = force_reg (SImode, operands[2]);
607 }
608 else if (MEM_P (operands[2]))
609 {
610 operands[2] = force_reg (SImode, operands[2]);
611 }
612
613 if (REG_P (operands[2]))
614 {
615 rtx tmp = gen_reg_rtx (SImode);
616 rtx tmp1 = gen_reg_rtx (<MODE>mode);
617 emit_insn (gen_negsi2 (tmp, operands[2]));
618 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
619 convert_to_mode (<VEL>mode,
620 tmp, 0)));
621 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
622 operands[1],
623 tmp1));
624 DONE;
625 }
626 else
627 FAIL;
628 }
629 )
630
631 (define_expand "ashr<mode>3"
632 [(match_operand:VDQ 0 "register_operand" "")
633 (match_operand:VDQ 1 "register_operand" "")
634 (match_operand:SI 2 "general_operand" "")]
635 "TARGET_SIMD"
636 {
637 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
638 int shift_amount;
639
640 if (CONST_INT_P (operands[2]))
641 {
642 shift_amount = INTVAL (operands[2]);
643 if (shift_amount > 0 && shift_amount <= bit_width)
644 {
645 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
646 shift_amount);
647 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
648 operands[1],
649 tmp));
650 DONE;
651 }
652 else
653 operands[2] = force_reg (SImode, operands[2]);
654 }
655 else if (MEM_P (operands[2]))
656 {
657 operands[2] = force_reg (SImode, operands[2]);
658 }
659
660 if (REG_P (operands[2]))
661 {
662 rtx tmp = gen_reg_rtx (SImode);
663 rtx tmp1 = gen_reg_rtx (<MODE>mode);
664 emit_insn (gen_negsi2 (tmp, operands[2]));
665 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
666 convert_to_mode (<VEL>mode,
667 tmp, 0)));
668 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
669 operands[1],
670 tmp1));
671 DONE;
672 }
673 else
674 FAIL;
675 }
676 )
677
678 (define_expand "vashl<mode>3"
679 [(match_operand:VDQ 0 "register_operand" "")
680 (match_operand:VDQ 1 "register_operand" "")
681 (match_operand:VDQ 2 "register_operand" "")]
682 "TARGET_SIMD"
683 {
684 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
685 operands[2]));
686 DONE;
687 })
688
689 ;; Using mode VQ_S as there is no V2DImode neg!
690 ;; Negating individual lanes most certainly offsets the
691 ;; gain from vectorization.
692 (define_expand "vashr<mode>3"
693 [(match_operand:VQ_S 0 "register_operand" "")
694 (match_operand:VQ_S 1 "register_operand" "")
695 (match_operand:VQ_S 2 "register_operand" "")]
696 "TARGET_SIMD"
697 {
698 rtx neg = gen_reg_rtx (<MODE>mode);
699 emit (gen_neg<mode>2 (neg, operands[2]));
700 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
701 neg));
702 DONE;
703 })
704
705 ;; DI vector shift
706 (define_expand "aarch64_ashr_simddi"
707 [(match_operand:DI 0 "register_operand" "=w")
708 (match_operand:DI 1 "register_operand" "w")
709 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
710 "TARGET_SIMD"
711 {
712 if (INTVAL (operands[2]) == 64)
713 emit_insn (gen_aarch64_sshr_simddi (operands[0], operands[1]));
714 else
715 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
716 DONE;
717 }
718 )
719
720 ;; SIMD shift by 64. This pattern is a special case as standard pattern does
721 ;; not handle NEON shifts by 64.
722 (define_insn "aarch64_sshr_simddi"
723 [(set (match_operand:DI 0 "register_operand" "=w")
724 (unspec:DI
725 [(match_operand:DI 1 "register_operand" "w")] UNSPEC_SSHR64))]
726 "TARGET_SIMD"
727 "sshr\t%d0, %d1, 64"
728 [(set_attr "type" "neon_shift_imm")]
729 )
730
731 (define_expand "vlshr<mode>3"
732 [(match_operand:VQ_S 0 "register_operand" "")
733 (match_operand:VQ_S 1 "register_operand" "")
734 (match_operand:VQ_S 2 "register_operand" "")]
735 "TARGET_SIMD"
736 {
737 rtx neg = gen_reg_rtx (<MODE>mode);
738 emit (gen_neg<mode>2 (neg, operands[2]));
739 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
740 neg));
741 DONE;
742 })
743
744 (define_expand "aarch64_lshr_simddi"
745 [(match_operand:DI 0 "register_operand" "=w")
746 (match_operand:DI 1 "register_operand" "w")
747 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
748 "TARGET_SIMD"
749 {
750 if (INTVAL (operands[2]) == 64)
751 emit_insn (gen_aarch64_ushr_simddi (operands[0], operands[1]));
752 else
753 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
754 DONE;
755 }
756 )
757
758 ;; SIMD shift by 64. This pattern is a special case as standard pattern does
759 ;; not handle NEON shifts by 64.
760 (define_insn "aarch64_ushr_simddi"
761 [(set (match_operand:DI 0 "register_operand" "=w")
762 (unspec:DI
763 [(match_operand:DI 1 "register_operand" "w")] UNSPEC_USHR64))]
764 "TARGET_SIMD"
765 "ushr\t%d0, %d1, 64"
766 [(set_attr "type" "neon_shift_imm")]
767 )
768
769 (define_expand "vec_set<mode>"
770 [(match_operand:VQ_S 0 "register_operand")
771 (match_operand:<VEL> 1 "register_operand")
772 (match_operand:SI 2 "immediate_operand")]
773 "TARGET_SIMD"
774 {
775 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
776 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
777 GEN_INT (elem), operands[0]));
778 DONE;
779 }
780 )
781
782 (define_insn "aarch64_simd_vec_setv2di"
783 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
784 (vec_merge:V2DI
785 (vec_duplicate:V2DI
786 (match_operand:DI 1 "register_operand" "r,w"))
787 (match_operand:V2DI 3 "register_operand" "0,0")
788 (match_operand:SI 2 "immediate_operand" "i,i")))]
789 "TARGET_SIMD"
790 {
791 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
792 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
793 switch (which_alternative)
794 {
795 case 0:
796 return "ins\\t%0.d[%p2], %1";
797 case 1:
798 return "ins\\t%0.d[%p2], %1.d[0]";
799 default:
800 gcc_unreachable ();
801 }
802 }
803 [(set_attr "type" "neon_from_gp, neon_ins_q")]
804 )
805
806 (define_expand "vec_setv2di"
807 [(match_operand:V2DI 0 "register_operand")
808 (match_operand:DI 1 "register_operand")
809 (match_operand:SI 2 "immediate_operand")]
810 "TARGET_SIMD"
811 {
812 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
813 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
814 GEN_INT (elem), operands[0]));
815 DONE;
816 }
817 )
818
819 (define_insn "aarch64_simd_vec_set<mode>"
820 [(set (match_operand:VDQF 0 "register_operand" "=w")
821 (vec_merge:VDQF
822 (vec_duplicate:VDQF
823 (match_operand:<VEL> 1 "register_operand" "w"))
824 (match_operand:VDQF 3 "register_operand" "0")
825 (match_operand:SI 2 "immediate_operand" "i")))]
826 "TARGET_SIMD"
827 {
828 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
829
830 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
831 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
832 }
833 [(set_attr "type" "neon_ins<q>")]
834 )
835
836 (define_expand "vec_set<mode>"
837 [(match_operand:VDQF 0 "register_operand" "+w")
838 (match_operand:<VEL> 1 "register_operand" "w")
839 (match_operand:SI 2 "immediate_operand" "")]
840 "TARGET_SIMD"
841 {
842 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
843 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
844 GEN_INT (elem), operands[0]));
845 DONE;
846 }
847 )
848
849
850 (define_insn "aarch64_mla<mode>"
851 [(set (match_operand:VQ_S 0 "register_operand" "=w")
852 (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
853 (match_operand:VQ_S 3 "register_operand" "w"))
854 (match_operand:VQ_S 1 "register_operand" "0")))]
855 "TARGET_SIMD"
856 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
857 [(set_attr "type" "neon_mla_<Vetype><q>")]
858 )
859
860 (define_insn "*aarch64_mla_elt<mode>"
861 [(set (match_operand:VDQHS 0 "register_operand" "=w")
862 (plus:VDQHS
863 (mult:VDQHS
864 (vec_duplicate:VDQHS
865 (vec_select:<VEL>
866 (match_operand:VDQHS 1 "register_operand" "<h_con>")
867 (parallel [(match_operand:SI 2 "immediate_operand")])))
868 (match_operand:VDQHS 3 "register_operand" "w"))
869 (match_operand:VDQHS 4 "register_operand" "0")))]
870 "TARGET_SIMD"
871 {
872 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
873 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
874 }
875 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
876 )
877
878 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
879 [(set (match_operand:VDQHS 0 "register_operand" "=w")
880 (plus:VDQHS
881 (mult:VDQHS
882 (vec_duplicate:VDQHS
883 (vec_select:<VEL>
884 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
885 (parallel [(match_operand:SI 2 "immediate_operand")])))
886 (match_operand:VDQHS 3 "register_operand" "w"))
887 (match_operand:VDQHS 4 "register_operand" "0")))]
888 "TARGET_SIMD"
889 {
890 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
891 INTVAL (operands[2])));
892 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
893 }
894 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
895 )
896
897 (define_insn "aarch64_mls<mode>"
898 [(set (match_operand:VQ_S 0 "register_operand" "=w")
899 (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
900 (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
901 (match_operand:VQ_S 3 "register_operand" "w"))))]
902 "TARGET_SIMD"
903 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
904 [(set_attr "type" "neon_mla_<Vetype><q>")]
905 )
906
907 (define_insn "*aarch64_mls_elt<mode>"
908 [(set (match_operand:VDQHS 0 "register_operand" "=w")
909 (minus:VDQHS
910 (match_operand:VDQHS 4 "register_operand" "0")
911 (mult:VDQHS
912 (vec_duplicate:VDQHS
913 (vec_select:<VEL>
914 (match_operand:VDQHS 1 "register_operand" "<h_con>")
915 (parallel [(match_operand:SI 2 "immediate_operand")])))
916 (match_operand:VDQHS 3 "register_operand" "w"))))]
917 "TARGET_SIMD"
918 {
919 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
920 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
921 }
922 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
923 )
924
925 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
926 [(set (match_operand:VDQHS 0 "register_operand" "=w")
927 (minus:VDQHS
928 (match_operand:VDQHS 4 "register_operand" "0")
929 (mult:VDQHS
930 (vec_duplicate:VDQHS
931 (vec_select:<VEL>
932 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
933 (parallel [(match_operand:SI 2 "immediate_operand")])))
934 (match_operand:VDQHS 3 "register_operand" "w"))))]
935 "TARGET_SIMD"
936 {
937 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
938 INTVAL (operands[2])));
939 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
940 }
941 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
942 )
943
944 ;; Max/Min operations.
945 (define_insn "<su><maxmin><mode>3"
946 [(set (match_operand:VQ_S 0 "register_operand" "=w")
947 (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
948 (match_operand:VQ_S 2 "register_operand" "w")))]
949 "TARGET_SIMD"
950 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
951 [(set_attr "type" "neon_minmax<q>")]
952 )
953
954 ;; vec_concat gives a new vector with the low elements from operand 1, and
955 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
956 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
957 ;; What that means, is that the RTL descriptions of the below patterns
958 ;; need to change depending on endianness.
959
960 ;; Move to the low architectural bits of the register.
961 ;; On little-endian this is { operand, zeroes }
962 ;; On big-endian this is { zeroes, operand }
963
964 (define_insn "move_lo_quad_internal_<mode>"
965 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
966 (vec_concat:VQ_NO2E
967 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
968 (vec_duplicate:<VHALF> (const_int 0))))]
969 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
970 "@
971 dup\\t%d0, %1.d[0]
972 fmov\\t%d0, %1
973 dup\\t%d0, %1"
974 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
975 (set_attr "simd" "yes,*,yes")
976 (set_attr "fp" "*,yes,*")
977 (set_attr "length" "4")]
978 )
979
980 (define_insn "move_lo_quad_internal_<mode>"
981 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
982 (vec_concat:VQ_2E
983 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
984 (const_int 0)))]
985 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
986 "@
987 dup\\t%d0, %1.d[0]
988 fmov\\t%d0, %1
989 dup\\t%d0, %1"
990 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
991 (set_attr "simd" "yes,*,yes")
992 (set_attr "fp" "*,yes,*")
993 (set_attr "length" "4")]
994 )
995
996 (define_insn "move_lo_quad_internal_be_<mode>"
997 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
998 (vec_concat:VQ_NO2E
999 (vec_duplicate:<VHALF> (const_int 0))
1000 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1001 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1002 "@
1003 dup\\t%d0, %1.d[0]
1004 fmov\\t%d0, %1
1005 dup\\t%d0, %1"
1006 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1007 (set_attr "simd" "yes,*,yes")
1008 (set_attr "fp" "*,yes,*")
1009 (set_attr "length" "4")]
1010 )
1011
1012 (define_insn "move_lo_quad_internal_be_<mode>"
1013 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1014 (vec_concat:VQ_2E
1015 (const_int 0)
1016 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1017 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1018 "@
1019 dup\\t%d0, %1.d[0]
1020 fmov\\t%d0, %1
1021 dup\\t%d0, %1"
1022 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1023 (set_attr "simd" "yes,*,yes")
1024 (set_attr "fp" "*,yes,*")
1025 (set_attr "length" "4")]
1026 )
1027
1028 (define_expand "move_lo_quad_<mode>"
1029 [(match_operand:VQ 0 "register_operand")
1030 (match_operand:VQ 1 "register_operand")]
1031 "TARGET_SIMD"
1032 {
1033 if (BYTES_BIG_ENDIAN)
1034 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1035 else
1036 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1037 DONE;
1038 }
1039 )
1040
1041 ;; Move operand1 to the high architectural bits of the register, keeping
1042 ;; the low architectural bits of operand2.
1043 ;; For little-endian this is { operand2, operand1 }
1044 ;; For big-endian this is { operand1, operand2 }
1045
1046 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1047 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1048 (vec_concat:VQ
1049 (vec_select:<VHALF>
1050 (match_dup 0)
1051 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1052 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1053 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1054 "@
1055 ins\\t%0.d[1], %1.d[0]
1056 ins\\t%0.d[1], %1"
1057 [(set_attr "type" "neon_ins")]
1058 )
1059
1060 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1061 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1062 (vec_concat:VQ
1063 (match_operand:<VHALF> 1 "register_operand" "w,r")
1064 (vec_select:<VHALF>
1065 (match_dup 0)
1066 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1067 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1068 "@
1069 ins\\t%0.d[1], %1.d[0]
1070 ins\\t%0.d[1], %1"
1071 [(set_attr "type" "neon_ins")]
1072 )
1073
1074 (define_expand "move_hi_quad_<mode>"
1075 [(match_operand:VQ 0 "register_operand" "")
1076 (match_operand:<VHALF> 1 "register_operand" "")]
1077 "TARGET_SIMD"
1078 {
1079 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1080 if (BYTES_BIG_ENDIAN)
1081 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1082 operands[1], p));
1083 else
1084 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1085 operands[1], p));
1086 DONE;
1087 })
1088
1089 ;; Narrowing operations.
1090
1091 ;; For doubles.
1092 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1093 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1094 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1095 "TARGET_SIMD"
1096 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1097 [(set_attr "type" "neon_shift_imm_narrow_q")]
1098 )
1099
1100 (define_expand "vec_pack_trunc_<mode>"
1101 [(match_operand:<VNARROWD> 0 "register_operand" "")
1102 (match_operand:VDN 1 "register_operand" "")
1103 (match_operand:VDN 2 "register_operand" "")]
1104 "TARGET_SIMD"
1105 {
1106 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1107 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1108 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1109
1110 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1111 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1112 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1113 DONE;
1114 })
1115
1116 ;; For quads.
1117
1118 (define_insn "vec_pack_trunc_<mode>"
1119 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1120 (vec_concat:<VNARROWQ2>
1121 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1122 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1123 "TARGET_SIMD"
1124 {
1125 if (BYTES_BIG_ENDIAN)
1126 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1127 else
1128 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1129 }
1130 [(set_attr "type" "multiple")
1131 (set_attr "length" "8")]
1132 )
1133
1134 ;; Widening operations.
1135
1136 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1137 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1138 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1139 (match_operand:VQW 1 "register_operand" "w")
1140 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1141 )))]
1142 "TARGET_SIMD"
1143 "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1144 [(set_attr "type" "neon_shift_imm_long")]
1145 )
1146
1147 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1148 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1149 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1150 (match_operand:VQW 1 "register_operand" "w")
1151 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1152 )))]
1153 "TARGET_SIMD"
1154 "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1155 [(set_attr "type" "neon_shift_imm_long")]
1156 )
1157
1158 (define_expand "vec_unpack<su>_hi_<mode>"
1159 [(match_operand:<VWIDE> 0 "register_operand" "")
1160 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1161 "TARGET_SIMD"
1162 {
1163 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1164 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1165 operands[1], p));
1166 DONE;
1167 }
1168 )
1169
1170 (define_expand "vec_unpack<su>_lo_<mode>"
1171 [(match_operand:<VWIDE> 0 "register_operand" "")
1172 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1173 "TARGET_SIMD"
1174 {
1175 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1176 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1177 operands[1], p));
1178 DONE;
1179 }
1180 )
1181
1182 ;; Widening arithmetic.
1183
1184 (define_insn "*aarch64_<su>mlal_lo<mode>"
1185 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1186 (plus:<VWIDE>
1187 (mult:<VWIDE>
1188 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1189 (match_operand:VQW 2 "register_operand" "w")
1190 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1191 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1192 (match_operand:VQW 4 "register_operand" "w")
1193 (match_dup 3))))
1194 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1195 "TARGET_SIMD"
1196 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1197 [(set_attr "type" "neon_mla_<Vetype>_long")]
1198 )
1199
1200 (define_insn "*aarch64_<su>mlal_hi<mode>"
1201 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1202 (plus:<VWIDE>
1203 (mult:<VWIDE>
1204 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1205 (match_operand:VQW 2 "register_operand" "w")
1206 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1207 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1208 (match_operand:VQW 4 "register_operand" "w")
1209 (match_dup 3))))
1210 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1211 "TARGET_SIMD"
1212 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1213 [(set_attr "type" "neon_mla_<Vetype>_long")]
1214 )
1215
1216 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1217 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1218 (minus:<VWIDE>
1219 (match_operand:<VWIDE> 1 "register_operand" "0")
1220 (mult:<VWIDE>
1221 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1222 (match_operand:VQW 2 "register_operand" "w")
1223 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1224 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1225 (match_operand:VQW 4 "register_operand" "w")
1226 (match_dup 3))))))]
1227 "TARGET_SIMD"
1228 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1229 [(set_attr "type" "neon_mla_<Vetype>_long")]
1230 )
1231
1232 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1233 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1234 (minus:<VWIDE>
1235 (match_operand:<VWIDE> 1 "register_operand" "0")
1236 (mult:<VWIDE>
1237 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1238 (match_operand:VQW 2 "register_operand" "w")
1239 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1240 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1241 (match_operand:VQW 4 "register_operand" "w")
1242 (match_dup 3))))))]
1243 "TARGET_SIMD"
1244 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1245 [(set_attr "type" "neon_mla_<Vetype>_long")]
1246 )
1247
1248 (define_insn "*aarch64_<su>mlal<mode>"
1249 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1250 (plus:<VWIDE>
1251 (mult:<VWIDE>
1252 (ANY_EXTEND:<VWIDE>
1253 (match_operand:VDW 1 "register_operand" "w"))
1254 (ANY_EXTEND:<VWIDE>
1255 (match_operand:VDW 2 "register_operand" "w")))
1256 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1257 "TARGET_SIMD"
1258 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1259 [(set_attr "type" "neon_mla_<Vetype>_long")]
1260 )
1261
1262 (define_insn "*aarch64_<su>mlsl<mode>"
1263 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1264 (minus:<VWIDE>
1265 (match_operand:<VWIDE> 1 "register_operand" "0")
1266 (mult:<VWIDE>
1267 (ANY_EXTEND:<VWIDE>
1268 (match_operand:VDW 2 "register_operand" "w"))
1269 (ANY_EXTEND:<VWIDE>
1270 (match_operand:VDW 3 "register_operand" "w")))))]
1271 "TARGET_SIMD"
1272 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1273 [(set_attr "type" "neon_mla_<Vetype>_long")]
1274 )
1275
1276 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1277 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1278 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1279 (match_operand:VQW 1 "register_operand" "w")
1280 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1281 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1282 (match_operand:VQW 2 "register_operand" "w")
1283 (match_dup 3)))))]
1284 "TARGET_SIMD"
1285 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1286 [(set_attr "type" "neon_mul_<Vetype>_long")]
1287 )
1288
1289 (define_expand "vec_widen_<su>mult_lo_<mode>"
1290 [(match_operand:<VWIDE> 0 "register_operand" "")
1291 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1292 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1293 "TARGET_SIMD"
1294 {
1295 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1296 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1297 operands[1],
1298 operands[2], p));
1299 DONE;
1300 }
1301 )
1302
1303 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1304 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1305 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1306 (match_operand:VQW 1 "register_operand" "w")
1307 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1308 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1309 (match_operand:VQW 2 "register_operand" "w")
1310 (match_dup 3)))))]
1311 "TARGET_SIMD"
1312 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1313 [(set_attr "type" "neon_mul_<Vetype>_long")]
1314 )
1315
1316 (define_expand "vec_widen_<su>mult_hi_<mode>"
1317 [(match_operand:<VWIDE> 0 "register_operand" "")
1318 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1319 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1320 "TARGET_SIMD"
1321 {
1322 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1323 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1324 operands[1],
1325 operands[2], p));
1326 DONE;
1327
1328 }
1329 )
1330
1331 ;; FP vector operations.
1332 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1333 ;; double-precision (64-bit) floating-point data types and arithmetic as
1334 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1335 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1336 ;;
1337 ;; Floating-point operations can raise an exception. Vectorizing such
1338 ;; operations are safe because of reasons explained below.
1339 ;;
1340 ;; ARMv8 permits an extension to enable trapped floating-point
1341 ;; exception handling, however this is an optional feature. In the
1342 ;; event of a floating-point exception being raised by vectorised
1343 ;; code then:
1344 ;; 1. If trapped floating-point exceptions are available, then a trap
1345 ;; will be taken when any lane raises an enabled exception. A trap
1346 ;; handler may determine which lane raised the exception.
1347 ;; 2. Alternatively a sticky exception flag is set in the
1348 ;; floating-point status register (FPSR). Software may explicitly
1349 ;; test the exception flags, in which case the tests will either
1350 ;; prevent vectorisation, allowing precise identification of the
1351 ;; failing operation, or if tested outside of vectorisable regions
1352 ;; then the specific operation and lane are not of interest.
1353
1354 ;; FP arithmetic operations.
1355
1356 (define_insn "add<mode>3"
1357 [(set (match_operand:VDQF 0 "register_operand" "=w")
1358 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1359 (match_operand:VDQF 2 "register_operand" "w")))]
1360 "TARGET_SIMD"
1361 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1362 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1363 )
1364
1365 (define_insn "sub<mode>3"
1366 [(set (match_operand:VDQF 0 "register_operand" "=w")
1367 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1368 (match_operand:VDQF 2 "register_operand" "w")))]
1369 "TARGET_SIMD"
1370 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1371 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1372 )
1373
1374 (define_insn "mul<mode>3"
1375 [(set (match_operand:VDQF 0 "register_operand" "=w")
1376 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1377 (match_operand:VDQF 2 "register_operand" "w")))]
1378 "TARGET_SIMD"
1379 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1380 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1381 )
1382
1383 (define_insn "div<mode>3"
1384 [(set (match_operand:VDQF 0 "register_operand" "=w")
1385 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1386 (match_operand:VDQF 2 "register_operand" "w")))]
1387 "TARGET_SIMD"
1388 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1389 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1390 )
1391
1392 (define_insn "neg<mode>2"
1393 [(set (match_operand:VDQF 0 "register_operand" "=w")
1394 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1395 "TARGET_SIMD"
1396 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1397 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1398 )
1399
1400 (define_insn "abs<mode>2"
1401 [(set (match_operand:VDQF 0 "register_operand" "=w")
1402 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1403 "TARGET_SIMD"
1404 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1405 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1406 )
1407
1408 (define_insn "fma<mode>4"
1409 [(set (match_operand:VDQF 0 "register_operand" "=w")
1410 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1411 (match_operand:VDQF 2 "register_operand" "w")
1412 (match_operand:VDQF 3 "register_operand" "0")))]
1413 "TARGET_SIMD"
1414 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1415 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1416 )
1417
1418 (define_insn "*aarch64_fma4_elt<mode>"
1419 [(set (match_operand:VDQF 0 "register_operand" "=w")
1420 (fma:VDQF
1421 (vec_duplicate:VDQF
1422 (vec_select:<VEL>
1423 (match_operand:VDQF 1 "register_operand" "<h_con>")
1424 (parallel [(match_operand:SI 2 "immediate_operand")])))
1425 (match_operand:VDQF 3 "register_operand" "w")
1426 (match_operand:VDQF 4 "register_operand" "0")))]
1427 "TARGET_SIMD"
1428 {
1429 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1430 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1431 }
1432 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1433 )
1434
1435 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1436 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1437 (fma:VDQSF
1438 (vec_duplicate:VDQSF
1439 (vec_select:<VEL>
1440 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1441 (parallel [(match_operand:SI 2 "immediate_operand")])))
1442 (match_operand:VDQSF 3 "register_operand" "w")
1443 (match_operand:VDQSF 4 "register_operand" "0")))]
1444 "TARGET_SIMD"
1445 {
1446 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1447 INTVAL (operands[2])));
1448 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1449 }
1450 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1451 )
1452
1453 (define_insn "*aarch64_fma4_elt_to_128df"
1454 [(set (match_operand:V2DF 0 "register_operand" "=w")
1455 (fma:V2DF
1456 (vec_duplicate:V2DF
1457 (match_operand:DF 1 "register_operand" "w"))
1458 (match_operand:V2DF 2 "register_operand" "w")
1459 (match_operand:V2DF 3 "register_operand" "0")))]
1460 "TARGET_SIMD"
1461 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1462 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1463 )
1464
1465 (define_insn "*aarch64_fma4_elt_to_64v2df"
1466 [(set (match_operand:DF 0 "register_operand" "=w")
1467 (fma:DF
1468 (vec_select:DF
1469 (match_operand:V2DF 1 "register_operand" "w")
1470 (parallel [(match_operand:SI 2 "immediate_operand")]))
1471 (match_operand:DF 3 "register_operand" "w")
1472 (match_operand:DF 4 "register_operand" "0")))]
1473 "TARGET_SIMD"
1474 {
1475 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1476 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1477 }
1478 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1479 )
1480
1481 (define_insn "fnma<mode>4"
1482 [(set (match_operand:VDQF 0 "register_operand" "=w")
1483 (fma:VDQF
1484 (match_operand:VDQF 1 "register_operand" "w")
1485 (neg:VDQF
1486 (match_operand:VDQF 2 "register_operand" "w"))
1487 (match_operand:VDQF 3 "register_operand" "0")))]
1488 "TARGET_SIMD"
1489 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1490 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1491 )
1492
1493 (define_insn "*aarch64_fnma4_elt<mode>"
1494 [(set (match_operand:VDQF 0 "register_operand" "=w")
1495 (fma:VDQF
1496 (neg:VDQF
1497 (match_operand:VDQF 3 "register_operand" "w"))
1498 (vec_duplicate:VDQF
1499 (vec_select:<VEL>
1500 (match_operand:VDQF 1 "register_operand" "<h_con>")
1501 (parallel [(match_operand:SI 2 "immediate_operand")])))
1502 (match_operand:VDQF 4 "register_operand" "0")))]
1503 "TARGET_SIMD"
1504 {
1505 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1506 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1507 }
1508 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1509 )
1510
1511 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1512 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1513 (fma:VDQSF
1514 (neg:VDQSF
1515 (match_operand:VDQSF 3 "register_operand" "w"))
1516 (vec_duplicate:VDQSF
1517 (vec_select:<VEL>
1518 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1519 (parallel [(match_operand:SI 2 "immediate_operand")])))
1520 (match_operand:VDQSF 4 "register_operand" "0")))]
1521 "TARGET_SIMD"
1522 {
1523 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1524 INTVAL (operands[2])));
1525 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1526 }
1527 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1528 )
1529
1530 (define_insn "*aarch64_fnma4_elt_to_128df"
1531 [(set (match_operand:V2DF 0 "register_operand" "=w")
1532 (fma:V2DF
1533 (neg:V2DF
1534 (match_operand:V2DF 2 "register_operand" "w"))
1535 (vec_duplicate:V2DF
1536 (match_operand:DF 1 "register_operand" "w"))
1537 (match_operand:V2DF 3 "register_operand" "0")))]
1538 "TARGET_SIMD"
1539 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1540 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1541 )
1542
1543 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1544 [(set (match_operand:DF 0 "register_operand" "=w")
1545 (fma:DF
1546 (vec_select:DF
1547 (match_operand:V2DF 1 "register_operand" "w")
1548 (parallel [(match_operand:SI 2 "immediate_operand")]))
1549 (neg:DF
1550 (match_operand:DF 3 "register_operand" "w"))
1551 (match_operand:DF 4 "register_operand" "0")))]
1552 "TARGET_SIMD"
1553 {
1554 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1555 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1556 }
1557 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1558 )
1559
1560 ;; Vector versions of the floating-point frint patterns.
1561 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1562 (define_insn "<frint_pattern><mode>2"
1563 [(set (match_operand:VDQF 0 "register_operand" "=w")
1564 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1565 FRINT))]
1566 "TARGET_SIMD"
1567 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1568 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1569 )
1570
1571 ;; Vector versions of the fcvt standard patterns.
1572 ;; Expands to lbtrunc, lround, lceil, lfloor
1573 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1574 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1575 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1576 [(match_operand:VDQF 1 "register_operand" "w")]
1577 FCVT)))]
1578 "TARGET_SIMD"
1579 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1580 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1581 )
1582
1583 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1584 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1585 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1586 [(match_operand:VDQF 1 "register_operand")]
1587 UNSPEC_FRINTZ)))]
1588 "TARGET_SIMD"
1589 {})
1590
1591 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1592 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1593 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1594 [(match_operand:VDQF 1 "register_operand")]
1595 UNSPEC_FRINTZ)))]
1596 "TARGET_SIMD"
1597 {})
1598
1599 (define_expand "ftrunc<VDQF:mode>2"
1600 [(set (match_operand:VDQF 0 "register_operand")
1601 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1602 UNSPEC_FRINTZ))]
1603 "TARGET_SIMD"
1604 {})
1605
1606 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1607 [(set (match_operand:VDQF 0 "register_operand" "=w")
1608 (FLOATUORS:VDQF
1609 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1610 "TARGET_SIMD"
1611 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1612 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1613 )
1614
1615 ;; Conversions between vectors of floats and doubles.
1616 ;; Contains a mix of patterns to match standard pattern names
1617 ;; and those for intrinsics.
1618
1619 ;; Float widening operations.
1620
1621 (define_insn "vec_unpacks_lo_v4sf"
1622 [(set (match_operand:V2DF 0 "register_operand" "=w")
1623 (float_extend:V2DF
1624 (vec_select:V2SF
1625 (match_operand:V4SF 1 "register_operand" "w")
1626 (parallel [(const_int 0) (const_int 1)])
1627 )))]
1628 "TARGET_SIMD"
1629 "fcvtl\\t%0.2d, %1.2s"
1630 [(set_attr "type" "neon_fp_cvt_widen_s")]
1631 )
1632
1633 (define_insn "aarch64_float_extend_lo_v2df"
1634 [(set (match_operand:V2DF 0 "register_operand" "=w")
1635 (float_extend:V2DF
1636 (match_operand:V2SF 1 "register_operand" "w")))]
1637 "TARGET_SIMD"
1638 "fcvtl\\t%0.2d, %1.2s"
1639 [(set_attr "type" "neon_fp_cvt_widen_s")]
1640 )
1641
1642 (define_insn "vec_unpacks_hi_v4sf"
1643 [(set (match_operand:V2DF 0 "register_operand" "=w")
1644 (float_extend:V2DF
1645 (vec_select:V2SF
1646 (match_operand:V4SF 1 "register_operand" "w")
1647 (parallel [(const_int 2) (const_int 3)])
1648 )))]
1649 "TARGET_SIMD"
1650 "fcvtl2\\t%0.2d, %1.4s"
1651 [(set_attr "type" "neon_fp_cvt_widen_s")]
1652 )
1653
1654 ;; Float narrowing operations.
1655
1656 (define_insn "aarch64_float_truncate_lo_v2sf"
1657 [(set (match_operand:V2SF 0 "register_operand" "=w")
1658 (float_truncate:V2SF
1659 (match_operand:V2DF 1 "register_operand" "w")))]
1660 "TARGET_SIMD"
1661 "fcvtn\\t%0.2s, %1.2d"
1662 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1663 )
1664
1665 (define_insn "aarch64_float_truncate_hi_v4sf"
1666 [(set (match_operand:V4SF 0 "register_operand" "=w")
1667 (vec_concat:V4SF
1668 (match_operand:V2SF 1 "register_operand" "0")
1669 (float_truncate:V2SF
1670 (match_operand:V2DF 2 "register_operand" "w"))))]
1671 "TARGET_SIMD"
1672 "fcvtn2\\t%0.4s, %2.2d"
1673 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1674 )
1675
1676 (define_expand "vec_pack_trunc_v2df"
1677 [(set (match_operand:V4SF 0 "register_operand")
1678 (vec_concat:V4SF
1679 (float_truncate:V2SF
1680 (match_operand:V2DF 1 "register_operand"))
1681 (float_truncate:V2SF
1682 (match_operand:V2DF 2 "register_operand"))
1683 ))]
1684 "TARGET_SIMD"
1685 {
1686 rtx tmp = gen_reg_rtx (V2SFmode);
1687 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1688 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1689
1690 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1691 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1692 tmp, operands[hi]));
1693 DONE;
1694 }
1695 )
1696
1697 (define_expand "vec_pack_trunc_df"
1698 [(set (match_operand:V2SF 0 "register_operand")
1699 (vec_concat:V2SF
1700 (float_truncate:SF
1701 (match_operand:DF 1 "register_operand"))
1702 (float_truncate:SF
1703 (match_operand:DF 2 "register_operand"))
1704 ))]
1705 "TARGET_SIMD"
1706 {
1707 rtx tmp = gen_reg_rtx (V2SFmode);
1708 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1709 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1710
1711 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1712 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1713 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1714 DONE;
1715 }
1716 )
1717
1718 (define_insn "aarch64_vmls<mode>"
1719 [(set (match_operand:VDQF 0 "register_operand" "=w")
1720 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1721 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1722 (match_operand:VDQF 3 "register_operand" "w"))))]
1723 "TARGET_SIMD"
1724 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1725 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1726 )
1727
1728 ;; FP Max/Min
1729 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1730 ;; expression like:
1731 ;; a = (b < c) ? b : c;
1732 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1733 ;; either explicitly or indirectly via -ffast-math.
1734 ;;
1735 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1736 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1737 ;; operand will be returned when both operands are zero (i.e. they may not
1738 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1739 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1740 ;; NaNs.
1741
1742 (define_insn "<su><maxmin><mode>3"
1743 [(set (match_operand:VDQF 0 "register_operand" "=w")
1744 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1745 (match_operand:VDQF 2 "register_operand" "w")))]
1746 "TARGET_SIMD"
1747 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1748 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1749 )
1750
1751 (define_insn "<maxmin_uns><mode>3"
1752 [(set (match_operand:VDQF 0 "register_operand" "=w")
1753 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1754 (match_operand:VDQF 2 "register_operand" "w")]
1755 FMAXMIN_UNS))]
1756 "TARGET_SIMD"
1757 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1758 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1759 )
1760
1761 ;; 'across lanes' add.
1762
1763 (define_expand "reduc_plus_scal_<mode>"
1764 [(match_operand:<VEL> 0 "register_operand" "=w")
1765 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")]
1766 UNSPEC_ADDV)]
1767 "TARGET_SIMD"
1768 {
1769 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1770 rtx scratch = gen_reg_rtx (<MODE>mode);
1771 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1772 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1773 DONE;
1774 }
1775 )
1776
1777 (define_expand "reduc_plus_scal_<mode>"
1778 [(match_operand:<VEL> 0 "register_operand" "=w")
1779 (match_operand:V2F 1 "register_operand" "w")]
1780 "TARGET_SIMD"
1781 {
1782 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1783 rtx scratch = gen_reg_rtx (<MODE>mode);
1784 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1785 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1786 DONE;
1787 }
1788 )
1789
1790 (define_insn "aarch64_reduc_plus_internal<mode>"
1791 [(set (match_operand:VDQV 0 "register_operand" "=w")
1792 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1793 UNSPEC_ADDV))]
1794 "TARGET_SIMD"
1795 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
1796 [(set_attr "type" "neon_reduc_add<q>")]
1797 )
1798
1799 (define_insn "aarch64_reduc_plus_internalv2si"
1800 [(set (match_operand:V2SI 0 "register_operand" "=w")
1801 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1802 UNSPEC_ADDV))]
1803 "TARGET_SIMD"
1804 "addp\\t%0.2s, %1.2s, %1.2s"
1805 [(set_attr "type" "neon_reduc_add")]
1806 )
1807
1808 (define_insn "aarch64_reduc_plus_internal<mode>"
1809 [(set (match_operand:V2F 0 "register_operand" "=w")
1810 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1811 UNSPEC_FADDV))]
1812 "TARGET_SIMD"
1813 "faddp\\t%<Vetype>0, %1.<Vtype>"
1814 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
1815 )
1816
1817 (define_insn "aarch64_addpv4sf"
1818 [(set (match_operand:V4SF 0 "register_operand" "=w")
1819 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1820 UNSPEC_FADDV))]
1821 "TARGET_SIMD"
1822 "faddp\\t%0.4s, %1.4s, %1.4s"
1823 [(set_attr "type" "neon_fp_reduc_add_s_q")]
1824 )
1825
1826 (define_expand "reduc_plus_scal_v4sf"
1827 [(set (match_operand:SF 0 "register_operand")
1828 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1829 UNSPEC_FADDV))]
1830 "TARGET_SIMD"
1831 {
1832 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
1833 rtx scratch = gen_reg_rtx (V4SFmode);
1834 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
1835 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
1836 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
1837 DONE;
1838 })
1839
1840 (define_insn "clz<mode>2"
1841 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1842 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1843 "TARGET_SIMD"
1844 "clz\\t%0.<Vtype>, %1.<Vtype>"
1845 [(set_attr "type" "neon_cls<q>")]
1846 )
1847
1848 ;; 'across lanes' max and min ops.
1849
1850 ;; Template for outputting a scalar, so we can create __builtins which can be
1851 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
1852 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1853 [(match_operand:<VEL> 0 "register_operand")
1854 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1855 FMAXMINV)]
1856 "TARGET_SIMD"
1857 {
1858 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1859 rtx scratch = gen_reg_rtx (<MODE>mode);
1860 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1861 operands[1]));
1862 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1863 DONE;
1864 }
1865 )
1866
1867 ;; Likewise for integer cases, signed and unsigned.
1868 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1869 [(match_operand:<VEL> 0 "register_operand")
1870 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
1871 MAXMINV)]
1872 "TARGET_SIMD"
1873 {
1874 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1875 rtx scratch = gen_reg_rtx (<MODE>mode);
1876 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1877 operands[1]));
1878 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1879 DONE;
1880 }
1881 )
1882
1883 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1884 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
1885 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
1886 MAXMINV))]
1887 "TARGET_SIMD"
1888 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1889 [(set_attr "type" "neon_reduc_minmax<q>")]
1890 )
1891
1892 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
1893 [(set (match_operand:V2SI 0 "register_operand" "=w")
1894 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1895 MAXMINV))]
1896 "TARGET_SIMD"
1897 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1898 [(set_attr "type" "neon_reduc_minmax")]
1899 )
1900
1901 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1902 [(set (match_operand:VDQF 0 "register_operand" "=w")
1903 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1904 FMAXMINV))]
1905 "TARGET_SIMD"
1906 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
1907 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
1908 )
1909
1910 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1911 ;; allocation.
1912 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1913 ;; to select.
1914 ;;
1915 ;; Thus our BSL is of the form:
1916 ;; op0 = bsl (mask, op2, op3)
1917 ;; We can use any of:
1918 ;;
1919 ;; if (op0 = mask)
1920 ;; bsl mask, op1, op2
1921 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1922 ;; bit op0, op2, mask
1923 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1924 ;; bif op0, op1, mask
1925
1926 (define_insn "aarch64_simd_bsl<mode>_internal"
1927 [(set (match_operand:VALLDIF 0 "register_operand" "=w,w,w")
1928 (ior:VALLDIF
1929 (and:VALLDIF
1930 (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w")
1931 (match_operand:VALLDIF 2 "register_operand" " w,w,0"))
1932 (and:VALLDIF
1933 (not:<V_cmp_result>
1934 (match_dup:<V_cmp_result> 1))
1935 (match_operand:VALLDIF 3 "register_operand" " w,0,w"))
1936 ))]
1937 "TARGET_SIMD"
1938 "@
1939 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1940 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1941 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1942 [(set_attr "type" "neon_bsl<q>")]
1943 )
1944
1945 (define_expand "aarch64_simd_bsl<mode>"
1946 [(match_operand:VALLDIF 0 "register_operand")
1947 (match_operand:<V_cmp_result> 1 "register_operand")
1948 (match_operand:VALLDIF 2 "register_operand")
1949 (match_operand:VALLDIF 3 "register_operand")]
1950 "TARGET_SIMD"
1951 {
1952 /* We can't alias operands together if they have different modes. */
1953 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1954 emit_insn (gen_aarch64_simd_bsl<mode>_internal (operands[0], operands[1],
1955 operands[2], operands[3]));
1956 DONE;
1957 })
1958
1959 (define_expand "aarch64_vcond_internal<mode><mode>"
1960 [(set (match_operand:VDQ 0 "register_operand")
1961 (if_then_else:VDQ
1962 (match_operator 3 "comparison_operator"
1963 [(match_operand:VDQ 4 "register_operand")
1964 (match_operand:VDQ 5 "nonmemory_operand")])
1965 (match_operand:VDQ 1 "nonmemory_operand")
1966 (match_operand:VDQ 2 "nonmemory_operand")))]
1967 "TARGET_SIMD"
1968 {
1969 rtx op1 = operands[1];
1970 rtx op2 = operands[2];
1971 rtx mask = gen_reg_rtx (<MODE>mode);
1972 enum rtx_code code = GET_CODE (operands[3]);
1973
1974 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
1975 and desirable for other comparisons if it results in FOO ? -1 : 0
1976 (this allows direct use of the comparison result without a bsl). */
1977 if (code == NE
1978 || (code != EQ
1979 && op1 == CONST0_RTX (<V_cmp_result>mode)
1980 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
1981 {
1982 op1 = operands[2];
1983 op2 = operands[1];
1984 switch (code)
1985 {
1986 case LE: code = GT; break;
1987 case LT: code = GE; break;
1988 case GE: code = LT; break;
1989 case GT: code = LE; break;
1990 /* No case EQ. */
1991 case NE: code = EQ; break;
1992 case LTU: code = GEU; break;
1993 case LEU: code = GTU; break;
1994 case GTU: code = LEU; break;
1995 case GEU: code = LTU; break;
1996 default: gcc_unreachable ();
1997 }
1998 }
1999
2000 /* Make sure we can handle the last operand. */
2001 switch (code)
2002 {
2003 case NE:
2004 /* Normalized to EQ above. */
2005 gcc_unreachable ();
2006
2007 case LE:
2008 case LT:
2009 case GE:
2010 case GT:
2011 case EQ:
2012 /* These instructions have a form taking an immediate zero. */
2013 if (operands[5] == CONST0_RTX (<MODE>mode))
2014 break;
2015 /* Fall through, as may need to load into register. */
2016 default:
2017 if (!REG_P (operands[5]))
2018 operands[5] = force_reg (<MODE>mode, operands[5]);
2019 break;
2020 }
2021
2022 switch (code)
2023 {
2024 case LT:
2025 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2026 break;
2027
2028 case GE:
2029 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2030 break;
2031
2032 case LE:
2033 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2034 break;
2035
2036 case GT:
2037 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2038 break;
2039
2040 case LTU:
2041 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2042 break;
2043
2044 case GEU:
2045 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2046 break;
2047
2048 case LEU:
2049 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2050 break;
2051
2052 case GTU:
2053 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2054 break;
2055
2056 /* NE has been normalized to EQ above. */
2057 case EQ:
2058 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2059 break;
2060
2061 default:
2062 gcc_unreachable ();
2063 }
2064
2065 /* If we have (a = (b CMP c) ? -1 : 0);
2066 Then we can simply move the generated mask. */
2067
2068 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2069 && op2 == CONST0_RTX (<V_cmp_result>mode))
2070 emit_move_insn (operands[0], mask);
2071 else
2072 {
2073 if (!REG_P (op1))
2074 op1 = force_reg (<MODE>mode, op1);
2075 if (!REG_P (op2))
2076 op2 = force_reg (<MODE>mode, op2);
2077 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2078 op1, op2));
2079 }
2080
2081 DONE;
2082 })
2083
2084 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2085 [(set (match_operand:VDQF_COND 0 "register_operand")
2086 (if_then_else:VDQF
2087 (match_operator 3 "comparison_operator"
2088 [(match_operand:VDQF 4 "register_operand")
2089 (match_operand:VDQF 5 "nonmemory_operand")])
2090 (match_operand:VDQF_COND 1 "nonmemory_operand")
2091 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2092 "TARGET_SIMD"
2093 {
2094 int inverse = 0;
2095 int use_zero_form = 0;
2096 int swap_bsl_operands = 0;
2097 rtx op1 = operands[1];
2098 rtx op2 = operands[2];
2099 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2100 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2101
2102 rtx (*base_comparison) (rtx, rtx, rtx);
2103 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2104
2105 switch (GET_CODE (operands[3]))
2106 {
2107 case GE:
2108 case GT:
2109 case LE:
2110 case LT:
2111 case EQ:
2112 if (operands[5] == CONST0_RTX (<MODE>mode))
2113 {
2114 use_zero_form = 1;
2115 break;
2116 }
2117 /* Fall through. */
2118 default:
2119 if (!REG_P (operands[5]))
2120 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2121 }
2122
2123 switch (GET_CODE (operands[3]))
2124 {
2125 case LT:
2126 case UNLT:
2127 inverse = 1;
2128 /* Fall through. */
2129 case GE:
2130 case UNGE:
2131 case ORDERED:
2132 case UNORDERED:
2133 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2134 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2135 break;
2136 case LE:
2137 case UNLE:
2138 inverse = 1;
2139 /* Fall through. */
2140 case GT:
2141 case UNGT:
2142 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2143 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2144 break;
2145 case EQ:
2146 case NE:
2147 case UNEQ:
2148 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2149 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2150 break;
2151 default:
2152 gcc_unreachable ();
2153 }
2154
2155 switch (GET_CODE (operands[3]))
2156 {
2157 case LT:
2158 case LE:
2159 case GT:
2160 case GE:
2161 case EQ:
2162 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2163 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2164 a GE b -> a GE b
2165 a GT b -> a GT b
2166 a LE b -> b GE a
2167 a LT b -> b GT a
2168 a EQ b -> a EQ b
2169 Note that there also exist direct comparison against 0 forms,
2170 so catch those as a special case. */
2171 if (use_zero_form)
2172 {
2173 inverse = 0;
2174 switch (GET_CODE (operands[3]))
2175 {
2176 case LT:
2177 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2178 break;
2179 case LE:
2180 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2181 break;
2182 default:
2183 /* Do nothing, other zero form cases already have the correct
2184 base_comparison. */
2185 break;
2186 }
2187 }
2188
2189 if (!inverse)
2190 emit_insn (base_comparison (mask, operands[4], operands[5]));
2191 else
2192 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2193 break;
2194 case UNLT:
2195 case UNLE:
2196 case UNGT:
2197 case UNGE:
2198 case NE:
2199 /* FCM returns false for lanes which are unordered, so if we use
2200 the inverse of the comparison we actually want to emit, then
2201 swap the operands to BSL, we will end up with the correct result.
2202 Note that a NE NaN and NaN NE b are true for all a, b.
2203
2204 Our transformations are:
2205 a GE b -> !(b GT a)
2206 a GT b -> !(b GE a)
2207 a LE b -> !(a GT b)
2208 a LT b -> !(a GE b)
2209 a NE b -> !(a EQ b) */
2210
2211 if (inverse)
2212 emit_insn (base_comparison (mask, operands[4], operands[5]));
2213 else
2214 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2215
2216 swap_bsl_operands = 1;
2217 break;
2218 case UNEQ:
2219 /* We check (a > b || b > a). combining these comparisons give us
2220 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2221 will then give us (a == b || a UNORDERED b) as intended. */
2222
2223 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2224 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2225 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2226 swap_bsl_operands = 1;
2227 break;
2228 case UNORDERED:
2229 /* Operands are ORDERED iff (a > b || b >= a).
2230 Swapping the operands to BSL will give the UNORDERED case. */
2231 swap_bsl_operands = 1;
2232 /* Fall through. */
2233 case ORDERED:
2234 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2235 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2236 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2237 break;
2238 default:
2239 gcc_unreachable ();
2240 }
2241
2242 if (swap_bsl_operands)
2243 {
2244 op1 = operands[2];
2245 op2 = operands[1];
2246 }
2247
2248 /* If we have (a = (b CMP c) ? -1 : 0);
2249 Then we can simply move the generated mask. */
2250
2251 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2252 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2253 emit_move_insn (operands[0], mask);
2254 else
2255 {
2256 if (!REG_P (op1))
2257 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2258 if (!REG_P (op2))
2259 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2260 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2261 op1, op2));
2262 }
2263
2264 DONE;
2265 })
2266
2267 (define_expand "vcond<mode><mode>"
2268 [(set (match_operand:VALL 0 "register_operand")
2269 (if_then_else:VALL
2270 (match_operator 3 "comparison_operator"
2271 [(match_operand:VALL 4 "register_operand")
2272 (match_operand:VALL 5 "nonmemory_operand")])
2273 (match_operand:VALL 1 "nonmemory_operand")
2274 (match_operand:VALL 2 "nonmemory_operand")))]
2275 "TARGET_SIMD"
2276 {
2277 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2278 operands[2], operands[3],
2279 operands[4], operands[5]));
2280 DONE;
2281 })
2282
2283 (define_expand "vcond<v_cmp_result><mode>"
2284 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2285 (if_then_else:<V_cmp_result>
2286 (match_operator 3 "comparison_operator"
2287 [(match_operand:VDQF 4 "register_operand")
2288 (match_operand:VDQF 5 "nonmemory_operand")])
2289 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2290 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2291 "TARGET_SIMD"
2292 {
2293 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2294 operands[0], operands[1],
2295 operands[2], operands[3],
2296 operands[4], operands[5]));
2297 DONE;
2298 })
2299
2300 (define_expand "vcondu<mode><mode>"
2301 [(set (match_operand:VDQ 0 "register_operand")
2302 (if_then_else:VDQ
2303 (match_operator 3 "comparison_operator"
2304 [(match_operand:VDQ 4 "register_operand")
2305 (match_operand:VDQ 5 "nonmemory_operand")])
2306 (match_operand:VDQ 1 "nonmemory_operand")
2307 (match_operand:VDQ 2 "nonmemory_operand")))]
2308 "TARGET_SIMD"
2309 {
2310 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2311 operands[2], operands[3],
2312 operands[4], operands[5]));
2313 DONE;
2314 })
2315
2316 ;; Patterns for AArch64 SIMD Intrinsics.
2317
2318 (define_expand "aarch64_create<mode>"
2319 [(match_operand:VD1 0 "register_operand" "")
2320 (match_operand:DI 1 "general_operand" "")]
2321 "TARGET_SIMD"
2322 {
2323 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2324 emit_move_insn (operands[0], src);
2325 DONE;
2326 })
2327
2328 ;; Lane extraction with sign extension to general purpose register.
2329 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2330 [(set (match_operand:GPI 0 "register_operand" "=r")
2331 (sign_extend:GPI
2332 (vec_select:<VEL>
2333 (match_operand:VDQQH 1 "register_operand" "w")
2334 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2335 "TARGET_SIMD"
2336 {
2337 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2338 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2339 }
2340 [(set_attr "type" "neon_to_gp<q>")]
2341 )
2342
2343 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2344 [(set (match_operand:SI 0 "register_operand" "=r")
2345 (zero_extend:SI
2346 (vec_select:<VEL>
2347 (match_operand:VDQQH 1 "register_operand" "w")
2348 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2349 "TARGET_SIMD"
2350 {
2351 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2352 return "umov\\t%w0, %1.<Vetype>[%2]";
2353 }
2354 [(set_attr "type" "neon_to_gp<q>")]
2355 )
2356
2357 (define_expand "aarch64_be_checked_get_lane<mode>"
2358 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand")
2359 (match_operand:VALL 1 "register_operand")
2360 (match_operand:SI 2 "immediate_operand")]
2361 "TARGET_SIMD"
2362 {
2363 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2364 emit_insn (gen_aarch64_get_lane<mode> (operands[0],
2365 operands[1],
2366 operands[2]));
2367 DONE;
2368 }
2369 )
2370
2371 ;; Lane extraction of a value, neither sign nor zero extension
2372 ;; is guaranteed so upper bits should be considered undefined.
2373 (define_insn "aarch64_get_lane<mode>"
2374 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2375 (vec_select:<VEL>
2376 (match_operand:VALL 1 "register_operand" "w, w, w")
2377 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2378 "TARGET_SIMD"
2379 {
2380 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2381 switch (which_alternative)
2382 {
2383 case 0:
2384 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2385 case 1:
2386 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2387 case 2:
2388 return "st1\\t{%1.<Vetype>}[%2], %0";
2389 default:
2390 gcc_unreachable ();
2391 }
2392 }
2393 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2394 )
2395
2396 (define_expand "aarch64_get_lanedi"
2397 [(match_operand:DI 0 "register_operand")
2398 (match_operand:DI 1 "register_operand")
2399 (match_operand:SI 2 "immediate_operand")]
2400 "TARGET_SIMD"
2401 {
2402 aarch64_simd_lane_bounds (operands[2], 0, 1);
2403 emit_move_insn (operands[0], operands[1]);
2404 DONE;
2405 })
2406
2407 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2408 ;; dest vector.
2409
2410 (define_insn "*aarch64_combinez<mode>"
2411 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2412 (vec_concat:<VDBL>
2413 (match_operand:VDIC 1 "register_operand" "w")
2414 (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2415 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2416 "mov\\t%0.8b, %1.8b"
2417 [(set_attr "type" "neon_move<q>")]
2418 )
2419
2420 (define_insn "*aarch64_combinez_be<mode>"
2421 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2422 (vec_concat:<VDBL>
2423 (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")
2424 (match_operand:VDIC 1 "register_operand" "w")))]
2425 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2426 "mov\\t%0.8b, %1.8b"
2427 [(set_attr "type" "neon_move<q>")]
2428 )
2429
2430 (define_expand "aarch64_combine<mode>"
2431 [(match_operand:<VDBL> 0 "register_operand")
2432 (match_operand:VDC 1 "register_operand")
2433 (match_operand:VDC 2 "register_operand")]
2434 "TARGET_SIMD"
2435 {
2436 rtx op1, op2;
2437 if (BYTES_BIG_ENDIAN)
2438 {
2439 op1 = operands[2];
2440 op2 = operands[1];
2441 }
2442 else
2443 {
2444 op1 = operands[1];
2445 op2 = operands[2];
2446 }
2447 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2448 DONE;
2449 }
2450 )
2451
2452 (define_insn_and_split "aarch64_combine_internal<mode>"
2453 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2454 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2455 (match_operand:VDC 2 "register_operand" "w")))]
2456 "TARGET_SIMD"
2457 "#"
2458 "&& reload_completed"
2459 [(const_int 0)]
2460 {
2461 if (BYTES_BIG_ENDIAN)
2462 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2463 else
2464 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2465 DONE;
2466 }
2467 [(set_attr "type" "multiple")]
2468 )
2469
2470 (define_expand "aarch64_simd_combine<mode>"
2471 [(match_operand:<VDBL> 0 "register_operand")
2472 (match_operand:VDC 1 "register_operand")
2473 (match_operand:VDC 2 "register_operand")]
2474 "TARGET_SIMD"
2475 {
2476 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2477 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2478 DONE;
2479 }
2480 [(set_attr "type" "multiple")]
2481 )
2482
2483 ;; <su><addsub>l<q>.
2484
2485 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2486 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2487 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2488 (match_operand:VQW 1 "register_operand" "w")
2489 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2490 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2491 (match_operand:VQW 2 "register_operand" "w")
2492 (match_dup 3)))))]
2493 "TARGET_SIMD"
2494 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2495 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2496 )
2497
2498 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2499 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2500 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2501 (match_operand:VQW 1 "register_operand" "w")
2502 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2503 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2504 (match_operand:VQW 2 "register_operand" "w")
2505 (match_dup 3)))))]
2506 "TARGET_SIMD"
2507 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2508 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2509 )
2510
2511
2512 (define_expand "aarch64_saddl2<mode>"
2513 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2514 (match_operand:VQW 1 "register_operand" "w")
2515 (match_operand:VQW 2 "register_operand" "w")]
2516 "TARGET_SIMD"
2517 {
2518 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2519 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2520 operands[2], p));
2521 DONE;
2522 })
2523
2524 (define_expand "aarch64_uaddl2<mode>"
2525 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2526 (match_operand:VQW 1 "register_operand" "w")
2527 (match_operand:VQW 2 "register_operand" "w")]
2528 "TARGET_SIMD"
2529 {
2530 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2531 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2532 operands[2], p));
2533 DONE;
2534 })
2535
2536 (define_expand "aarch64_ssubl2<mode>"
2537 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2538 (match_operand:VQW 1 "register_operand" "w")
2539 (match_operand:VQW 2 "register_operand" "w")]
2540 "TARGET_SIMD"
2541 {
2542 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2543 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2544 operands[2], p));
2545 DONE;
2546 })
2547
2548 (define_expand "aarch64_usubl2<mode>"
2549 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2550 (match_operand:VQW 1 "register_operand" "w")
2551 (match_operand:VQW 2 "register_operand" "w")]
2552 "TARGET_SIMD"
2553 {
2554 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2555 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2556 operands[2], p));
2557 DONE;
2558 })
2559
2560 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2561 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2562 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2563 (match_operand:VDW 1 "register_operand" "w"))
2564 (ANY_EXTEND:<VWIDE>
2565 (match_operand:VDW 2 "register_operand" "w"))))]
2566 "TARGET_SIMD"
2567 "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2568 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2569 )
2570
2571 ;; <su><addsub>w<q>.
2572
2573 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2574 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2575 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2576 (ANY_EXTEND:<VWIDE>
2577 (match_operand:VDW 2 "register_operand" "w"))))]
2578 "TARGET_SIMD"
2579 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2580 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2581 )
2582
2583 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2584 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2585 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2586 (ANY_EXTEND:<VWIDE>
2587 (vec_select:<VHALF>
2588 (match_operand:VQW 2 "register_operand" "w")
2589 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2590 "TARGET_SIMD"
2591 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2592 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2593 )
2594
2595 (define_expand "aarch64_saddw2<mode>"
2596 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2597 (match_operand:<VWIDE> 1 "register_operand" "w")
2598 (match_operand:VQW 2 "register_operand" "w")]
2599 "TARGET_SIMD"
2600 {
2601 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2602 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2603 operands[2], p));
2604 DONE;
2605 })
2606
2607 (define_expand "aarch64_uaddw2<mode>"
2608 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2609 (match_operand:<VWIDE> 1 "register_operand" "w")
2610 (match_operand:VQW 2 "register_operand" "w")]
2611 "TARGET_SIMD"
2612 {
2613 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2614 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2615 operands[2], p));
2616 DONE;
2617 })
2618
2619
2620 (define_expand "aarch64_ssubw2<mode>"
2621 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2622 (match_operand:<VWIDE> 1 "register_operand" "w")
2623 (match_operand:VQW 2 "register_operand" "w")]
2624 "TARGET_SIMD"
2625 {
2626 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2627 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2628 operands[2], p));
2629 DONE;
2630 })
2631
2632 (define_expand "aarch64_usubw2<mode>"
2633 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2634 (match_operand:<VWIDE> 1 "register_operand" "w")
2635 (match_operand:VQW 2 "register_operand" "w")]
2636 "TARGET_SIMD"
2637 {
2638 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2639 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2640 operands[2], p));
2641 DONE;
2642 })
2643
2644 ;; <su><r>h<addsub>.
2645
2646 (define_insn "aarch64_<sur>h<addsub><mode>"
2647 [(set (match_operand:VQ_S 0 "register_operand" "=w")
2648 (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2649 (match_operand:VQ_S 2 "register_operand" "w")]
2650 HADDSUB))]
2651 "TARGET_SIMD"
2652 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2653 [(set_attr "type" "neon_<addsub>_halve<q>")]
2654 )
2655
2656 ;; <r><addsub>hn<q>.
2657
2658 (define_insn "aarch64_<sur><addsub>hn<mode>"
2659 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2660 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2661 (match_operand:VQN 2 "register_operand" "w")]
2662 ADDSUBHN))]
2663 "TARGET_SIMD"
2664 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2665 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2666 )
2667
2668 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2669 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2670 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2671 (match_operand:VQN 2 "register_operand" "w")
2672 (match_operand:VQN 3 "register_operand" "w")]
2673 ADDSUBHN2))]
2674 "TARGET_SIMD"
2675 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2676 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2677 )
2678
2679 ;; pmul.
2680
2681 (define_insn "aarch64_pmul<mode>"
2682 [(set (match_operand:VB 0 "register_operand" "=w")
2683 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2684 (match_operand:VB 2 "register_operand" "w")]
2685 UNSPEC_PMUL))]
2686 "TARGET_SIMD"
2687 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2688 [(set_attr "type" "neon_mul_<Vetype><q>")]
2689 )
2690
2691 ;; <su>q<addsub>
2692
2693 (define_insn "aarch64_<su_optab><optab><mode>"
2694 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2695 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2696 (match_operand:VSDQ_I 2 "register_operand" "w")))]
2697 "TARGET_SIMD"
2698 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2699 [(set_attr "type" "neon_<optab><q>")]
2700 )
2701
2702 ;; suqadd and usqadd
2703
2704 (define_insn "aarch64_<sur>qadd<mode>"
2705 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2706 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2707 (match_operand:VSDQ_I 2 "register_operand" "w")]
2708 USSUQADD))]
2709 "TARGET_SIMD"
2710 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2711 [(set_attr "type" "neon_qadd<q>")]
2712 )
2713
2714 ;; sqmovun
2715
2716 (define_insn "aarch64_sqmovun<mode>"
2717 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2718 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2719 UNSPEC_SQXTUN))]
2720 "TARGET_SIMD"
2721 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2722 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2723 )
2724
2725 ;; sqmovn and uqmovn
2726
2727 (define_insn "aarch64_<sur>qmovn<mode>"
2728 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2729 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2730 SUQMOVN))]
2731 "TARGET_SIMD"
2732 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2733 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2734 )
2735
2736 ;; <su>q<absneg>
2737
2738 (define_insn "aarch64_s<optab><mode>"
2739 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2740 (UNQOPS:VSDQ_I
2741 (match_operand:VSDQ_I 1 "register_operand" "w")))]
2742 "TARGET_SIMD"
2743 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2744 [(set_attr "type" "neon_<optab><q>")]
2745 )
2746
2747 ;; sq<r>dmulh.
2748
2749 (define_insn "aarch64_sq<r>dmulh<mode>"
2750 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2751 (unspec:VSDQ_HSI
2752 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2753 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2754 VQDMULH))]
2755 "TARGET_SIMD"
2756 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2757 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
2758 )
2759
2760 ;; sq<r>dmulh_lane
2761
2762 (define_expand "aarch64_sqdmulh_lane<mode>"
2763 [(match_operand:VDQHS 0 "register_operand" "")
2764 (match_operand:VDQHS 1 "register_operand" "")
2765 (match_operand:<VCOND> 2 "register_operand" "")
2766 (match_operand:SI 3 "immediate_operand" "")]
2767 "TARGET_SIMD"
2768 {
2769 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2770 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2771 emit_insn (gen_aarch64_sqdmulh_lane<mode>_internal (operands[0],
2772 operands[1],
2773 operands[2],
2774 operands[3]));
2775 DONE;
2776 }
2777 )
2778
2779 (define_expand "aarch64_sqrdmulh_lane<mode>"
2780 [(match_operand:VDQHS 0 "register_operand" "")
2781 (match_operand:VDQHS 1 "register_operand" "")
2782 (match_operand:<VCOND> 2 "register_operand" "")
2783 (match_operand:SI 3 "immediate_operand" "")]
2784 "TARGET_SIMD"
2785 {
2786 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2787 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2788 emit_insn (gen_aarch64_sqrdmulh_lane<mode>_internal (operands[0],
2789 operands[1],
2790 operands[2],
2791 operands[3]));
2792 DONE;
2793 }
2794 )
2795
2796 (define_insn "aarch64_sq<r>dmulh_lane<mode>_internal"
2797 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2798 (unspec:VDQHS
2799 [(match_operand:VDQHS 1 "register_operand" "w")
2800 (vec_select:<VEL>
2801 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2802 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2803 VQDMULH))]
2804 "TARGET_SIMD"
2805 "*
2806 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2807 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2808 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2809 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2810 )
2811
2812 (define_expand "aarch64_sqdmulh_laneq<mode>"
2813 [(match_operand:VSDQ_HSI 0 "register_operand" "")
2814 (match_operand:VSDQ_HSI 1 "register_operand" "")
2815 (match_operand:<VCONQ> 2 "register_operand" "")
2816 (match_operand:SI 3 "immediate_operand" "")]
2817 "TARGET_SIMD"
2818 {
2819 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2820 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2821 emit_insn (gen_aarch64_sqdmulh_laneq<mode>_internal (operands[0],
2822 operands[1],
2823 operands[2],
2824 operands[3]));
2825 DONE;
2826 }
2827 )
2828
2829 (define_expand "aarch64_sqrdmulh_laneq<mode>"
2830 [(match_operand:VSDQ_HSI 0 "register_operand" "")
2831 (match_operand:VSDQ_HSI 1 "register_operand" "")
2832 (match_operand:<VCONQ> 2 "register_operand" "")
2833 (match_operand:SI 3 "immediate_operand" "")]
2834 "TARGET_SIMD"
2835 {
2836 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2837 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2838 emit_insn (gen_aarch64_sqrdmulh_laneq<mode>_internal (operands[0],
2839 operands[1],
2840 operands[2],
2841 operands[3]));
2842 DONE;
2843 }
2844 )
2845
2846 (define_insn "aarch64_sq<r>dmulh_laneq<mode>_internal"
2847 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2848 (unspec:VDQHS
2849 [(match_operand:VDQHS 1 "register_operand" "w")
2850 (vec_select:<VEL>
2851 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2852 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2853 VQDMULH))]
2854 "TARGET_SIMD"
2855 "*
2856 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2857 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2858 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2859 )
2860
2861 (define_expand "aarch64_sqdmulh_lane<mode>"
2862 [(match_operand:SD_HSI 0 "register_operand" "")
2863 (match_operand:SD_HSI 1 "register_operand" "")
2864 (match_operand:<VCOND> 2 "register_operand" "")
2865 (match_operand:SI 3 "immediate_operand" "")]
2866 "TARGET_SIMD"
2867 {
2868 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2869 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2870 emit_insn (gen_aarch64_sqdmulh_lane<mode>_internal (operands[0],
2871 operands[1],
2872 operands[2],
2873 operands[3]));
2874 DONE;
2875 }
2876 )
2877
2878 (define_expand "aarch64_sqrdmulh_lane<mode>"
2879 [(match_operand:SD_HSI 0 "register_operand" "")
2880 (match_operand:SD_HSI 1 "register_operand" "")
2881 (match_operand:<VCOND> 2 "register_operand" "")
2882 (match_operand:SI 3 "immediate_operand" "")]
2883 "TARGET_SIMD"
2884 {
2885 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2886 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2887 emit_insn (gen_aarch64_sqrdmulh_lane<mode>_internal (operands[0],
2888 operands[1],
2889 operands[2],
2890 operands[3]));
2891 DONE;
2892 }
2893 )
2894
2895 (define_insn "aarch64_sq<r>dmulh_lane<mode>_internal"
2896 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2897 (unspec:SD_HSI
2898 [(match_operand:SD_HSI 1 "register_operand" "w")
2899 (vec_select:<VEL>
2900 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2901 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2902 VQDMULH))]
2903 "TARGET_SIMD"
2904 "*
2905 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2906 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2907 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2908 )
2909
2910 (define_insn "aarch64_sq<r>dmulh_laneq<mode>_internal"
2911 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2912 (unspec:SD_HSI
2913 [(match_operand:SD_HSI 1 "register_operand" "w")
2914 (vec_select:<VEL>
2915 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2916 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2917 VQDMULH))]
2918 "TARGET_SIMD"
2919 "*
2920 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2921 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2922 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2923 )
2924
2925 ;; vqdml[sa]l
2926
2927 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2928 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2929 (SBINQOPS:<VWIDE>
2930 (match_operand:<VWIDE> 1 "register_operand" "0")
2931 (ss_ashift:<VWIDE>
2932 (mult:<VWIDE>
2933 (sign_extend:<VWIDE>
2934 (match_operand:VSD_HSI 2 "register_operand" "w"))
2935 (sign_extend:<VWIDE>
2936 (match_operand:VSD_HSI 3 "register_operand" "w")))
2937 (const_int 1))))]
2938 "TARGET_SIMD"
2939 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2940 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
2941 )
2942
2943 ;; vqdml[sa]l_lane
2944
2945 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2946 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2947 (SBINQOPS:<VWIDE>
2948 (match_operand:<VWIDE> 1 "register_operand" "0")
2949 (ss_ashift:<VWIDE>
2950 (mult:<VWIDE>
2951 (sign_extend:<VWIDE>
2952 (match_operand:VD_HSI 2 "register_operand" "w"))
2953 (sign_extend:<VWIDE>
2954 (vec_duplicate:VD_HSI
2955 (vec_select:<VEL>
2956 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2957 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2958 ))
2959 (const_int 1))))]
2960 "TARGET_SIMD"
2961 {
2962 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2963 return
2964 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2965 }
2966 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2967 )
2968
2969 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>_internal"
2970 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2971 (SBINQOPS:<VWIDE>
2972 (match_operand:<VWIDE> 1 "register_operand" "0")
2973 (ss_ashift:<VWIDE>
2974 (mult:<VWIDE>
2975 (sign_extend:<VWIDE>
2976 (match_operand:VD_HSI 2 "register_operand" "w"))
2977 (sign_extend:<VWIDE>
2978 (vec_duplicate:VD_HSI
2979 (vec_select:<VEL>
2980 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
2981 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2982 ))
2983 (const_int 1))))]
2984 "TARGET_SIMD"
2985 {
2986 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2987 return
2988 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2989 }
2990 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2991 )
2992
2993 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2994 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2995 (SBINQOPS:<VWIDE>
2996 (match_operand:<VWIDE> 1 "register_operand" "0")
2997 (ss_ashift:<VWIDE>
2998 (mult:<VWIDE>
2999 (sign_extend:<VWIDE>
3000 (match_operand:SD_HSI 2 "register_operand" "w"))
3001 (sign_extend:<VWIDE>
3002 (vec_select:<VEL>
3003 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3004 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3005 )
3006 (const_int 1))))]
3007 "TARGET_SIMD"
3008 {
3009 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3010 return
3011 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3012 }
3013 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3014 )
3015
3016 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>_internal"
3017 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3018 (SBINQOPS:<VWIDE>
3019 (match_operand:<VWIDE> 1 "register_operand" "0")
3020 (ss_ashift:<VWIDE>
3021 (mult:<VWIDE>
3022 (sign_extend:<VWIDE>
3023 (match_operand:SD_HSI 2 "register_operand" "w"))
3024 (sign_extend:<VWIDE>
3025 (vec_select:<VEL>
3026 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3027 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3028 )
3029 (const_int 1))))]
3030 "TARGET_SIMD"
3031 {
3032 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3033 return
3034 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3035 }
3036 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3037 )
3038
3039 (define_expand "aarch64_sqdmlal_lane<mode>"
3040 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3041 (match_operand:<VWIDE> 1 "register_operand" "0")
3042 (match_operand:VSD_HSI 2 "register_operand" "w")
3043 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3044 (match_operand:SI 4 "immediate_operand" "i")]
3045 "TARGET_SIMD"
3046 {
3047 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3048 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3049 emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
3050 operands[2], operands[3],
3051 operands[4]));
3052 DONE;
3053 })
3054
3055 (define_expand "aarch64_sqdmlal_laneq<mode>"
3056 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3057 (match_operand:<VWIDE> 1 "register_operand" "0")
3058 (match_operand:VSD_HSI 2 "register_operand" "w")
3059 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3060 (match_operand:SI 4 "immediate_operand" "i")]
3061 "TARGET_SIMD"
3062 {
3063 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3064 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3065 emit_insn (gen_aarch64_sqdmlal_laneq<mode>_internal (operands[0], operands[1],
3066 operands[2], operands[3],
3067 operands[4]));
3068 DONE;
3069 })
3070
3071 (define_expand "aarch64_sqdmlsl_lane<mode>"
3072 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3073 (match_operand:<VWIDE> 1 "register_operand" "0")
3074 (match_operand:VSD_HSI 2 "register_operand" "w")
3075 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3076 (match_operand:SI 4 "immediate_operand" "i")]
3077 "TARGET_SIMD"
3078 {
3079 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3080 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3081 emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
3082 operands[2], operands[3],
3083 operands[4]));
3084 DONE;
3085 })
3086
3087 (define_expand "aarch64_sqdmlsl_laneq<mode>"
3088 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3089 (match_operand:<VWIDE> 1 "register_operand" "0")
3090 (match_operand:VSD_HSI 2 "register_operand" "w")
3091 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3092 (match_operand:SI 4 "immediate_operand" "i")]
3093 "TARGET_SIMD"
3094 {
3095 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3096 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3097 emit_insn (gen_aarch64_sqdmlsl_laneq<mode>_internal (operands[0], operands[1],
3098 operands[2], operands[3],
3099 operands[4]));
3100 DONE;
3101 })
3102
3103 ;; vqdml[sa]l_n
3104
3105 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3106 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3107 (SBINQOPS:<VWIDE>
3108 (match_operand:<VWIDE> 1 "register_operand" "0")
3109 (ss_ashift:<VWIDE>
3110 (mult:<VWIDE>
3111 (sign_extend:<VWIDE>
3112 (match_operand:VD_HSI 2 "register_operand" "w"))
3113 (sign_extend:<VWIDE>
3114 (vec_duplicate:VD_HSI
3115 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3116 (const_int 1))))]
3117 "TARGET_SIMD"
3118 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3119 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3120 )
3121
3122 ;; sqdml[as]l2
3123
3124 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3125 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3126 (SBINQOPS:<VWIDE>
3127 (match_operand:<VWIDE> 1 "register_operand" "0")
3128 (ss_ashift:<VWIDE>
3129 (mult:<VWIDE>
3130 (sign_extend:<VWIDE>
3131 (vec_select:<VHALF>
3132 (match_operand:VQ_HSI 2 "register_operand" "w")
3133 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3134 (sign_extend:<VWIDE>
3135 (vec_select:<VHALF>
3136 (match_operand:VQ_HSI 3 "register_operand" "w")
3137 (match_dup 4))))
3138 (const_int 1))))]
3139 "TARGET_SIMD"
3140 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3141 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3142 )
3143
3144 (define_expand "aarch64_sqdmlal2<mode>"
3145 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3146 (match_operand:<VWIDE> 1 "register_operand" "w")
3147 (match_operand:VQ_HSI 2 "register_operand" "w")
3148 (match_operand:VQ_HSI 3 "register_operand" "w")]
3149 "TARGET_SIMD"
3150 {
3151 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3152 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3153 operands[2], operands[3], p));
3154 DONE;
3155 })
3156
3157 (define_expand "aarch64_sqdmlsl2<mode>"
3158 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3159 (match_operand:<VWIDE> 1 "register_operand" "w")
3160 (match_operand:VQ_HSI 2 "register_operand" "w")
3161 (match_operand:VQ_HSI 3 "register_operand" "w")]
3162 "TARGET_SIMD"
3163 {
3164 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3165 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3166 operands[2], operands[3], p));
3167 DONE;
3168 })
3169
3170 ;; vqdml[sa]l2_lane
3171
3172 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3173 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3174 (SBINQOPS:<VWIDE>
3175 (match_operand:<VWIDE> 1 "register_operand" "0")
3176 (ss_ashift:<VWIDE>
3177 (mult:<VWIDE>
3178 (sign_extend:<VWIDE>
3179 (vec_select:<VHALF>
3180 (match_operand:VQ_HSI 2 "register_operand" "w")
3181 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3182 (sign_extend:<VWIDE>
3183 (vec_duplicate:<VHALF>
3184 (vec_select:<VEL>
3185 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3186 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3187 ))))
3188 (const_int 1))))]
3189 "TARGET_SIMD"
3190 {
3191 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3192 return
3193 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3194 }
3195 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3196 )
3197
3198 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3199 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3200 (SBINQOPS:<VWIDE>
3201 (match_operand:<VWIDE> 1 "register_operand" "0")
3202 (ss_ashift:<VWIDE>
3203 (mult:<VWIDE>
3204 (sign_extend:<VWIDE>
3205 (vec_select:<VHALF>
3206 (match_operand:VQ_HSI 2 "register_operand" "w")
3207 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3208 (sign_extend:<VWIDE>
3209 (vec_duplicate:<VHALF>
3210 (vec_select:<VEL>
3211 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3212 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3213 ))))
3214 (const_int 1))))]
3215 "TARGET_SIMD"
3216 {
3217 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3218 return
3219 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3220 }
3221 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3222 )
3223
3224 (define_expand "aarch64_sqdmlal2_lane<mode>"
3225 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3226 (match_operand:<VWIDE> 1 "register_operand" "w")
3227 (match_operand:VQ_HSI 2 "register_operand" "w")
3228 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3229 (match_operand:SI 4 "immediate_operand" "i")]
3230 "TARGET_SIMD"
3231 {
3232 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3233 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3234 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3235 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3236 operands[2], operands[3],
3237 operands[4], p));
3238 DONE;
3239 })
3240
3241 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3242 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3243 (match_operand:<VWIDE> 1 "register_operand" "w")
3244 (match_operand:VQ_HSI 2 "register_operand" "w")
3245 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3246 (match_operand:SI 4 "immediate_operand" "i")]
3247 "TARGET_SIMD"
3248 {
3249 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3250 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3251 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3252 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3253 operands[2], operands[3],
3254 operands[4], p));
3255 DONE;
3256 })
3257
3258 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3259 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3260 (match_operand:<VWIDE> 1 "register_operand" "w")
3261 (match_operand:VQ_HSI 2 "register_operand" "w")
3262 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3263 (match_operand:SI 4 "immediate_operand" "i")]
3264 "TARGET_SIMD"
3265 {
3266 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3267 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3268 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3269 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3270 operands[2], operands[3],
3271 operands[4], p));
3272 DONE;
3273 })
3274
3275 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3276 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3277 (match_operand:<VWIDE> 1 "register_operand" "w")
3278 (match_operand:VQ_HSI 2 "register_operand" "w")
3279 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3280 (match_operand:SI 4 "immediate_operand" "i")]
3281 "TARGET_SIMD"
3282 {
3283 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3284 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3285 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3286 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3287 operands[2], operands[3],
3288 operands[4], p));
3289 DONE;
3290 })
3291
3292 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3293 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3294 (SBINQOPS:<VWIDE>
3295 (match_operand:<VWIDE> 1 "register_operand" "0")
3296 (ss_ashift:<VWIDE>
3297 (mult:<VWIDE>
3298 (sign_extend:<VWIDE>
3299 (vec_select:<VHALF>
3300 (match_operand:VQ_HSI 2 "register_operand" "w")
3301 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3302 (sign_extend:<VWIDE>
3303 (vec_duplicate:<VHALF>
3304 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3305 (const_int 1))))]
3306 "TARGET_SIMD"
3307 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3308 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3309 )
3310
3311 (define_expand "aarch64_sqdmlal2_n<mode>"
3312 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3313 (match_operand:<VWIDE> 1 "register_operand" "w")
3314 (match_operand:VQ_HSI 2 "register_operand" "w")
3315 (match_operand:<VEL> 3 "register_operand" "w")]
3316 "TARGET_SIMD"
3317 {
3318 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3319 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3320 operands[2], operands[3],
3321 p));
3322 DONE;
3323 })
3324
3325 (define_expand "aarch64_sqdmlsl2_n<mode>"
3326 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3327 (match_operand:<VWIDE> 1 "register_operand" "w")
3328 (match_operand:VQ_HSI 2 "register_operand" "w")
3329 (match_operand:<VEL> 3 "register_operand" "w")]
3330 "TARGET_SIMD"
3331 {
3332 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3333 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3334 operands[2], operands[3],
3335 p));
3336 DONE;
3337 })
3338
3339 ;; vqdmull
3340
3341 (define_insn "aarch64_sqdmull<mode>"
3342 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3343 (ss_ashift:<VWIDE>
3344 (mult:<VWIDE>
3345 (sign_extend:<VWIDE>
3346 (match_operand:VSD_HSI 1 "register_operand" "w"))
3347 (sign_extend:<VWIDE>
3348 (match_operand:VSD_HSI 2 "register_operand" "w")))
3349 (const_int 1)))]
3350 "TARGET_SIMD"
3351 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3352 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3353 )
3354
3355 ;; vqdmull_lane
3356
3357 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3358 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3359 (ss_ashift:<VWIDE>
3360 (mult:<VWIDE>
3361 (sign_extend:<VWIDE>
3362 (match_operand:VD_HSI 1 "register_operand" "w"))
3363 (sign_extend:<VWIDE>
3364 (vec_duplicate:VD_HSI
3365 (vec_select:<VEL>
3366 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3367 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3368 ))
3369 (const_int 1)))]
3370 "TARGET_SIMD"
3371 {
3372 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3373 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3374 }
3375 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3376 )
3377
3378 (define_insn "aarch64_sqdmull_laneq<mode>_internal"
3379 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3380 (ss_ashift:<VWIDE>
3381 (mult:<VWIDE>
3382 (sign_extend:<VWIDE>
3383 (match_operand:VD_HSI 1 "register_operand" "w"))
3384 (sign_extend:<VWIDE>
3385 (vec_duplicate:VD_HSI
3386 (vec_select:<VEL>
3387 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3388 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3389 ))
3390 (const_int 1)))]
3391 "TARGET_SIMD"
3392 {
3393 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3394 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3395 }
3396 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3397 )
3398
3399 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3400 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3401 (ss_ashift:<VWIDE>
3402 (mult:<VWIDE>
3403 (sign_extend:<VWIDE>
3404 (match_operand:SD_HSI 1 "register_operand" "w"))
3405 (sign_extend:<VWIDE>
3406 (vec_select:<VEL>
3407 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3408 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3409 ))
3410 (const_int 1)))]
3411 "TARGET_SIMD"
3412 {
3413 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3414 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3415 }
3416 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3417 )
3418
3419 (define_insn "aarch64_sqdmull_laneq<mode>_internal"
3420 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3421 (ss_ashift:<VWIDE>
3422 (mult:<VWIDE>
3423 (sign_extend:<VWIDE>
3424 (match_operand:SD_HSI 1 "register_operand" "w"))
3425 (sign_extend:<VWIDE>
3426 (vec_select:<VEL>
3427 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3428 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3429 ))
3430 (const_int 1)))]
3431 "TARGET_SIMD"
3432 {
3433 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3434 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3435 }
3436 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3437 )
3438
3439 (define_expand "aarch64_sqdmull_lane<mode>"
3440 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3441 (match_operand:VSD_HSI 1 "register_operand" "w")
3442 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3443 (match_operand:SI 3 "immediate_operand" "i")]
3444 "TARGET_SIMD"
3445 {
3446 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
3447 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3448 emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
3449 operands[2], operands[3]));
3450 DONE;
3451 })
3452
3453 (define_expand "aarch64_sqdmull_laneq<mode>"
3454 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3455 (match_operand:VSD_HSI 1 "register_operand" "w")
3456 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3457 (match_operand:SI 3 "immediate_operand" "i")]
3458 "TARGET_SIMD"
3459 {
3460 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
3461 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3462 emit_insn (gen_aarch64_sqdmull_laneq<mode>_internal
3463 (operands[0], operands[1], operands[2], operands[3]));
3464 DONE;
3465 })
3466
3467 ;; vqdmull_n
3468
3469 (define_insn "aarch64_sqdmull_n<mode>"
3470 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3471 (ss_ashift:<VWIDE>
3472 (mult:<VWIDE>
3473 (sign_extend:<VWIDE>
3474 (match_operand:VD_HSI 1 "register_operand" "w"))
3475 (sign_extend:<VWIDE>
3476 (vec_duplicate:VD_HSI
3477 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3478 )
3479 (const_int 1)))]
3480 "TARGET_SIMD"
3481 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3482 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3483 )
3484
3485 ;; vqdmull2
3486
3487
3488
3489 (define_insn "aarch64_sqdmull2<mode>_internal"
3490 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3491 (ss_ashift:<VWIDE>
3492 (mult:<VWIDE>
3493 (sign_extend:<VWIDE>
3494 (vec_select:<VHALF>
3495 (match_operand:VQ_HSI 1 "register_operand" "w")
3496 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3497 (sign_extend:<VWIDE>
3498 (vec_select:<VHALF>
3499 (match_operand:VQ_HSI 2 "register_operand" "w")
3500 (match_dup 3)))
3501 )
3502 (const_int 1)))]
3503 "TARGET_SIMD"
3504 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3505 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3506 )
3507
3508 (define_expand "aarch64_sqdmull2<mode>"
3509 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3510 (match_operand:VQ_HSI 1 "register_operand" "w")
3511 (match_operand:VQ_HSI 2 "register_operand" "w")]
3512 "TARGET_SIMD"
3513 {
3514 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3515 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3516 operands[2], p));
3517 DONE;
3518 })
3519
3520 ;; vqdmull2_lane
3521
3522 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3523 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3524 (ss_ashift:<VWIDE>
3525 (mult:<VWIDE>
3526 (sign_extend:<VWIDE>
3527 (vec_select:<VHALF>
3528 (match_operand:VQ_HSI 1 "register_operand" "w")
3529 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3530 (sign_extend:<VWIDE>
3531 (vec_duplicate:<VHALF>
3532 (vec_select:<VEL>
3533 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3534 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3535 ))
3536 (const_int 1)))]
3537 "TARGET_SIMD"
3538 {
3539 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3540 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3541 }
3542 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3543 )
3544
3545 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3546 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3547 (ss_ashift:<VWIDE>
3548 (mult:<VWIDE>
3549 (sign_extend:<VWIDE>
3550 (vec_select:<VHALF>
3551 (match_operand:VQ_HSI 1 "register_operand" "w")
3552 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3553 (sign_extend:<VWIDE>
3554 (vec_duplicate:<VHALF>
3555 (vec_select:<VEL>
3556 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3557 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3558 ))
3559 (const_int 1)))]
3560 "TARGET_SIMD"
3561 {
3562 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3563 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3564 }
3565 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3566 )
3567
3568 (define_expand "aarch64_sqdmull2_lane<mode>"
3569 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3570 (match_operand:VQ_HSI 1 "register_operand" "w")
3571 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3572 (match_operand:SI 3 "immediate_operand" "i")]
3573 "TARGET_SIMD"
3574 {
3575 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3576 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
3577 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3578 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3579 operands[2], operands[3],
3580 p));
3581 DONE;
3582 })
3583
3584 (define_expand "aarch64_sqdmull2_laneq<mode>"
3585 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3586 (match_operand:VQ_HSI 1 "register_operand" "w")
3587 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3588 (match_operand:SI 3 "immediate_operand" "i")]
3589 "TARGET_SIMD"
3590 {
3591 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3592 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
3593 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3594 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3595 operands[2], operands[3],
3596 p));
3597 DONE;
3598 })
3599
3600 ;; vqdmull2_n
3601
3602 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3603 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3604 (ss_ashift:<VWIDE>
3605 (mult:<VWIDE>
3606 (sign_extend:<VWIDE>
3607 (vec_select:<VHALF>
3608 (match_operand:VQ_HSI 1 "register_operand" "w")
3609 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3610 (sign_extend:<VWIDE>
3611 (vec_duplicate:<VHALF>
3612 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3613 )
3614 (const_int 1)))]
3615 "TARGET_SIMD"
3616 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3617 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3618 )
3619
3620 (define_expand "aarch64_sqdmull2_n<mode>"
3621 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3622 (match_operand:VQ_HSI 1 "register_operand" "w")
3623 (match_operand:<VEL> 2 "register_operand" "w")]
3624 "TARGET_SIMD"
3625 {
3626 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3627 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3628 operands[2], p));
3629 DONE;
3630 })
3631
3632 ;; vshl
3633
3634 (define_insn "aarch64_<sur>shl<mode>"
3635 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3636 (unspec:VSDQ_I_DI
3637 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3638 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3639 VSHL))]
3640 "TARGET_SIMD"
3641 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3642 [(set_attr "type" "neon_shift_reg<q>")]
3643 )
3644
3645
3646 ;; vqshl
3647
3648 (define_insn "aarch64_<sur>q<r>shl<mode>"
3649 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3650 (unspec:VSDQ_I
3651 [(match_operand:VSDQ_I 1 "register_operand" "w")
3652 (match_operand:VSDQ_I 2 "register_operand" "w")]
3653 VQSHL))]
3654 "TARGET_SIMD"
3655 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3656 [(set_attr "type" "neon_sat_shift_reg<q>")]
3657 )
3658
3659 ;; vshll_n
3660
3661 (define_insn "aarch64_<sur>shll_n<mode>"
3662 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3663 (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3664 (match_operand:SI 2
3665 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
3666 VSHLL))]
3667 "TARGET_SIMD"
3668 "*
3669 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3670 if (INTVAL (operands[2]) == bit_width)
3671 {
3672 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3673 }
3674 else {
3675 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3676 }"
3677 [(set_attr "type" "neon_shift_imm_long")]
3678 )
3679
3680 ;; vshll_high_n
3681
3682 (define_insn "aarch64_<sur>shll2_n<mode>"
3683 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3684 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3685 (match_operand:SI 2 "immediate_operand" "i")]
3686 VSHLL))]
3687 "TARGET_SIMD"
3688 "*
3689 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3690 if (INTVAL (operands[2]) == bit_width)
3691 {
3692 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3693 }
3694 else {
3695 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3696 }"
3697 [(set_attr "type" "neon_shift_imm_long")]
3698 )
3699
3700 ;; vrshr_n
3701
3702 (define_insn "aarch64_<sur>shr_n<mode>"
3703 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3704 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3705 (match_operand:SI 2
3706 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3707 VRSHR_N))]
3708 "TARGET_SIMD"
3709 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3710 [(set_attr "type" "neon_sat_shift_imm<q>")]
3711 )
3712
3713 ;; v(r)sra_n
3714
3715 (define_insn "aarch64_<sur>sra_n<mode>"
3716 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3717 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3718 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3719 (match_operand:SI 3
3720 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3721 VSRA))]
3722 "TARGET_SIMD"
3723 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3724 [(set_attr "type" "neon_shift_acc<q>")]
3725 )
3726
3727 ;; vs<lr>i_n
3728
3729 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3730 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3731 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3732 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3733 (match_operand:SI 3
3734 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
3735 VSLRI))]
3736 "TARGET_SIMD"
3737 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3738 [(set_attr "type" "neon_shift_imm<q>")]
3739 )
3740
3741 ;; vqshl(u)
3742
3743 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3744 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3745 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3746 (match_operand:SI 2
3747 "aarch64_simd_shift_imm_<ve_mode>" "i")]
3748 VQSHL_N))]
3749 "TARGET_SIMD"
3750 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3751 [(set_attr "type" "neon_sat_shift_imm<q>")]
3752 )
3753
3754
3755 ;; vq(r)shr(u)n_n
3756
3757 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3758 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3759 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3760 (match_operand:SI 2
3761 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3762 VQSHRN_N))]
3763 "TARGET_SIMD"
3764 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
3765 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3766 )
3767
3768
3769 ;; cm(eq|ge|gt|lt|le)
3770 ;; Note, we have constraints for Dz and Z as different expanders
3771 ;; have different ideas of what should be passed to this pattern.
3772
3773 (define_insn "aarch64_cm<optab><mode>"
3774 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3775 (neg:<V_cmp_result>
3776 (COMPARISONS:<V_cmp_result>
3777 (match_operand:VDQ 1 "register_operand" "w,w")
3778 (match_operand:VDQ 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3779 )))]
3780 "TARGET_SIMD"
3781 "@
3782 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3783 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3784 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
3785 )
3786
3787 (define_insn_and_split "aarch64_cm<optab>di"
3788 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3789 (neg:DI
3790 (COMPARISONS:DI
3791 (match_operand:DI 1 "register_operand" "w,w,r")
3792 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3793 )))
3794 (clobber (reg:CC CC_REGNUM))]
3795 "TARGET_SIMD"
3796 "#"
3797 "reload_completed"
3798 [(set (match_operand:DI 0 "register_operand")
3799 (neg:DI
3800 (COMPARISONS:DI
3801 (match_operand:DI 1 "register_operand")
3802 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3803 )))]
3804 {
3805 /* If we are in the general purpose register file,
3806 we split to a sequence of comparison and store. */
3807 if (GP_REGNUM_P (REGNO (operands[0]))
3808 && GP_REGNUM_P (REGNO (operands[1])))
3809 {
3810 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3811 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3812 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3813 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3814 DONE;
3815 }
3816 /* Otherwise, we expand to a similar pattern which does not
3817 clobber CC_REGNUM. */
3818 }
3819 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
3820 )
3821
3822 (define_insn "*aarch64_cm<optab>di"
3823 [(set (match_operand:DI 0 "register_operand" "=w,w")
3824 (neg:DI
3825 (COMPARISONS:DI
3826 (match_operand:DI 1 "register_operand" "w,w")
3827 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3828 )))]
3829 "TARGET_SIMD && reload_completed"
3830 "@
3831 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3832 cm<optab>\t%d0, %d1, #0"
3833 [(set_attr "type" "neon_compare, neon_compare_zero")]
3834 )
3835
3836 ;; cm(hs|hi)
3837
3838 (define_insn "aarch64_cm<optab><mode>"
3839 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3840 (neg:<V_cmp_result>
3841 (UCOMPARISONS:<V_cmp_result>
3842 (match_operand:VDQ 1 "register_operand" "w")
3843 (match_operand:VDQ 2 "register_operand" "w")
3844 )))]
3845 "TARGET_SIMD"
3846 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3847 [(set_attr "type" "neon_compare<q>")]
3848 )
3849
3850 (define_insn_and_split "aarch64_cm<optab>di"
3851 [(set (match_operand:DI 0 "register_operand" "=w,r")
3852 (neg:DI
3853 (UCOMPARISONS:DI
3854 (match_operand:DI 1 "register_operand" "w,r")
3855 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3856 )))
3857 (clobber (reg:CC CC_REGNUM))]
3858 "TARGET_SIMD"
3859 "#"
3860 "reload_completed"
3861 [(set (match_operand:DI 0 "register_operand")
3862 (neg:DI
3863 (UCOMPARISONS:DI
3864 (match_operand:DI 1 "register_operand")
3865 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3866 )))]
3867 {
3868 /* If we are in the general purpose register file,
3869 we split to a sequence of comparison and store. */
3870 if (GP_REGNUM_P (REGNO (operands[0]))
3871 && GP_REGNUM_P (REGNO (operands[1])))
3872 {
3873 machine_mode mode = CCmode;
3874 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3875 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3876 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3877 DONE;
3878 }
3879 /* Otherwise, we expand to a similar pattern which does not
3880 clobber CC_REGNUM. */
3881 }
3882 [(set_attr "type" "neon_compare,multiple")]
3883 )
3884
3885 (define_insn "*aarch64_cm<optab>di"
3886 [(set (match_operand:DI 0 "register_operand" "=w")
3887 (neg:DI
3888 (UCOMPARISONS:DI
3889 (match_operand:DI 1 "register_operand" "w")
3890 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
3891 )))]
3892 "TARGET_SIMD && reload_completed"
3893 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
3894 [(set_attr "type" "neon_compare")]
3895 )
3896
3897 ;; cmtst
3898
3899 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
3900 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
3901 ;; not (neg (eq (and x y) 0))
3902 ;; which is rewritten by simplify_rtx as
3903 ;; plus (eq (and x y) 0) -1.
3904
3905 (define_insn "aarch64_cmtst<mode>"
3906 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3907 (plus:<V_cmp_result>
3908 (eq:<V_cmp_result>
3909 (and:VDQ
3910 (match_operand:VDQ 1 "register_operand" "w")
3911 (match_operand:VDQ 2 "register_operand" "w"))
3912 (match_operand:VDQ 3 "aarch64_simd_imm_zero"))
3913 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
3914 ]
3915 "TARGET_SIMD"
3916 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3917 [(set_attr "type" "neon_tst<q>")]
3918 )
3919
3920 (define_insn_and_split "aarch64_cmtstdi"
3921 [(set (match_operand:DI 0 "register_operand" "=w,r")
3922 (neg:DI
3923 (ne:DI
3924 (and:DI
3925 (match_operand:DI 1 "register_operand" "w,r")
3926 (match_operand:DI 2 "register_operand" "w,r"))
3927 (const_int 0))))
3928 (clobber (reg:CC CC_REGNUM))]
3929 "TARGET_SIMD"
3930 "#"
3931 "reload_completed"
3932 [(set (match_operand:DI 0 "register_operand")
3933 (neg:DI
3934 (ne:DI
3935 (and:DI
3936 (match_operand:DI 1 "register_operand")
3937 (match_operand:DI 2 "register_operand"))
3938 (const_int 0))))]
3939 {
3940 /* If we are in the general purpose register file,
3941 we split to a sequence of comparison and store. */
3942 if (GP_REGNUM_P (REGNO (operands[0]))
3943 && GP_REGNUM_P (REGNO (operands[1])))
3944 {
3945 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3946 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3947 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3948 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3949 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3950 DONE;
3951 }
3952 /* Otherwise, we expand to a similar pattern which does not
3953 clobber CC_REGNUM. */
3954 }
3955 [(set_attr "type" "neon_tst,multiple")]
3956 )
3957
3958 (define_insn "*aarch64_cmtstdi"
3959 [(set (match_operand:DI 0 "register_operand" "=w")
3960 (neg:DI
3961 (ne:DI
3962 (and:DI
3963 (match_operand:DI 1 "register_operand" "w")
3964 (match_operand:DI 2 "register_operand" "w"))
3965 (const_int 0))))]
3966 "TARGET_SIMD"
3967 "cmtst\t%d0, %d1, %d2"
3968 [(set_attr "type" "neon_tst")]
3969 )
3970
3971 ;; fcm(eq|ge|gt|le|lt)
3972
3973 (define_insn "aarch64_cm<optab><mode>"
3974 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3975 (neg:<V_cmp_result>
3976 (COMPARISONS:<V_cmp_result>
3977 (match_operand:VALLF 1 "register_operand" "w,w")
3978 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3979 )))]
3980 "TARGET_SIMD"
3981 "@
3982 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3983 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3984 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
3985 )
3986
3987 ;; fac(ge|gt)
3988 ;; Note we can also handle what would be fac(le|lt) by
3989 ;; generating fac(ge|gt).
3990
3991 (define_insn "*aarch64_fac<optab><mode>"
3992 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3993 (neg:<V_cmp_result>
3994 (FAC_COMPARISONS:<V_cmp_result>
3995 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3996 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3997 )))]
3998 "TARGET_SIMD"
3999 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4000 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4001 )
4002
4003 ;; addp
4004
4005 (define_insn "aarch64_addp<mode>"
4006 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4007 (unspec:VD_BHSI
4008 [(match_operand:VD_BHSI 1 "register_operand" "w")
4009 (match_operand:VD_BHSI 2 "register_operand" "w")]
4010 UNSPEC_ADDP))]
4011 "TARGET_SIMD"
4012 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4013 [(set_attr "type" "neon_reduc_add<q>")]
4014 )
4015
4016 (define_insn "aarch64_addpdi"
4017 [(set (match_operand:DI 0 "register_operand" "=w")
4018 (unspec:DI
4019 [(match_operand:V2DI 1 "register_operand" "w")]
4020 UNSPEC_ADDP))]
4021 "TARGET_SIMD"
4022 "addp\t%d0, %1.2d"
4023 [(set_attr "type" "neon_reduc_add")]
4024 )
4025
4026 ;; sqrt
4027
4028 (define_insn "sqrt<mode>2"
4029 [(set (match_operand:VDQF 0 "register_operand" "=w")
4030 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
4031 "TARGET_SIMD"
4032 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4033 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
4034 )
4035
4036 ;; Patterns for vector struct loads and stores.
4037
4038 (define_insn "vec_load_lanesoi<mode>"
4039 [(set (match_operand:OI 0 "register_operand" "=w")
4040 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4041 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4042 UNSPEC_LD2))]
4043 "TARGET_SIMD"
4044 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4045 [(set_attr "type" "neon_load2_2reg<q>")]
4046 )
4047
4048 (define_insn "aarch64_simd_ld2r<mode>"
4049 [(set (match_operand:OI 0 "register_operand" "=w")
4050 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4051 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4052 UNSPEC_LD2_DUP))]
4053 "TARGET_SIMD"
4054 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4055 [(set_attr "type" "neon_load2_all_lanes<q>")]
4056 )
4057
4058 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4059 [(set (match_operand:OI 0 "register_operand" "=w")
4060 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4061 (match_operand:OI 2 "register_operand" "0")
4062 (match_operand:SI 3 "immediate_operand" "i")
4063 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4064 UNSPEC_LD2_LANE))]
4065 "TARGET_SIMD"
4066 "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1"
4067 [(set_attr "type" "neon_load2_one_lane")]
4068 )
4069
4070 (define_insn "vec_store_lanesoi<mode>"
4071 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4072 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4073 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4074 UNSPEC_ST2))]
4075 "TARGET_SIMD"
4076 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4077 [(set_attr "type" "neon_store2_2reg<q>")]
4078 )
4079
4080 (define_insn "vec_store_lanesoi_lane<mode>"
4081 [(set (match_operand:<V_TWO_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4082 (unspec:<V_TWO_ELEM> [(match_operand:OI 1 "register_operand" "w")
4083 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4084 (match_operand:SI 2 "immediate_operand" "i")]
4085 UNSPEC_ST2_LANE))]
4086 "TARGET_SIMD"
4087 "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0"
4088 [(set_attr "type" "neon_store3_one_lane<q>")]
4089 )
4090
4091 (define_insn "vec_load_lanesci<mode>"
4092 [(set (match_operand:CI 0 "register_operand" "=w")
4093 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4094 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4095 UNSPEC_LD3))]
4096 "TARGET_SIMD"
4097 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4098 [(set_attr "type" "neon_load3_3reg<q>")]
4099 )
4100
4101 (define_insn "aarch64_simd_ld3r<mode>"
4102 [(set (match_operand:CI 0 "register_operand" "=w")
4103 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4104 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4105 UNSPEC_LD3_DUP))]
4106 "TARGET_SIMD"
4107 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4108 [(set_attr "type" "neon_load3_all_lanes<q>")]
4109 )
4110
4111 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4112 [(set (match_operand:CI 0 "register_operand" "=w")
4113 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4114 (match_operand:CI 2 "register_operand" "0")
4115 (match_operand:SI 3 "immediate_operand" "i")
4116 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4117 UNSPEC_LD3_LANE))]
4118 "TARGET_SIMD"
4119 "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1"
4120 [(set_attr "type" "neon_load3_one_lane")]
4121 )
4122
4123 (define_insn "vec_store_lanesci<mode>"
4124 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4125 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4126 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4127 UNSPEC_ST3))]
4128 "TARGET_SIMD"
4129 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4130 [(set_attr "type" "neon_store3_3reg<q>")]
4131 )
4132
4133 (define_insn "vec_store_lanesci_lane<mode>"
4134 [(set (match_operand:<V_THREE_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4135 (unspec:<V_THREE_ELEM> [(match_operand:CI 1 "register_operand" "w")
4136 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4137 (match_operand:SI 2 "immediate_operand" "i")]
4138 UNSPEC_ST3_LANE))]
4139 "TARGET_SIMD"
4140 "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0"
4141 [(set_attr "type" "neon_store3_one_lane<q>")]
4142 )
4143
4144 (define_insn "vec_load_lanesxi<mode>"
4145 [(set (match_operand:XI 0 "register_operand" "=w")
4146 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4147 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4148 UNSPEC_LD4))]
4149 "TARGET_SIMD"
4150 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4151 [(set_attr "type" "neon_load4_4reg<q>")]
4152 )
4153
4154 (define_insn "aarch64_simd_ld4r<mode>"
4155 [(set (match_operand:XI 0 "register_operand" "=w")
4156 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4157 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4158 UNSPEC_LD4_DUP))]
4159 "TARGET_SIMD"
4160 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4161 [(set_attr "type" "neon_load4_all_lanes<q>")]
4162 )
4163
4164 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4165 [(set (match_operand:XI 0 "register_operand" "=w")
4166 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4167 (match_operand:XI 2 "register_operand" "0")
4168 (match_operand:SI 3 "immediate_operand" "i")
4169 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4170 UNSPEC_LD4_LANE))]
4171 "TARGET_SIMD"
4172 "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1"
4173 [(set_attr "type" "neon_load4_one_lane")]
4174 )
4175
4176 (define_insn "vec_store_lanesxi<mode>"
4177 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4178 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4179 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4180 UNSPEC_ST4))]
4181 "TARGET_SIMD"
4182 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4183 [(set_attr "type" "neon_store4_4reg<q>")]
4184 )
4185
4186 (define_insn "vec_store_lanesxi_lane<mode>"
4187 [(set (match_operand:<V_FOUR_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4188 (unspec:<V_FOUR_ELEM> [(match_operand:XI 1 "register_operand" "w")
4189 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4190 (match_operand:SI 2 "immediate_operand" "i")]
4191 UNSPEC_ST4_LANE))]
4192 "TARGET_SIMD"
4193 "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0"
4194 [(set_attr "type" "neon_store4_one_lane<q>")]
4195 )
4196
4197 ;; Reload patterns for AdvSIMD register list operands.
4198
4199 (define_expand "mov<mode>"
4200 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
4201 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
4202 "TARGET_SIMD"
4203 {
4204 if (can_create_pseudo_p ())
4205 {
4206 if (GET_CODE (operands[0]) != REG)
4207 operands[1] = force_reg (<MODE>mode, operands[1]);
4208 }
4209 })
4210
4211 (define_insn "*aarch64_mov<mode>"
4212 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4213 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4214 "TARGET_SIMD
4215 && (register_operand (operands[0], <MODE>mode)
4216 || register_operand (operands[1], <MODE>mode))"
4217
4218 {
4219 switch (which_alternative)
4220 {
4221 case 0: return "#";
4222 case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
4223 case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
4224 default: gcc_unreachable ();
4225 }
4226 }
4227 [(set_attr "type" "neon_move,neon_store<nregs>_<nregs>reg_q,\
4228 neon_load<nregs>_<nregs>reg_q")
4229 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4230 )
4231
4232 (define_insn "aarch64_be_ld1<mode>"
4233 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4234 (unspec:VALLDI [(match_operand:VALLDI 1 "aarch64_simd_struct_operand" "Utv")]
4235 UNSPEC_LD1))]
4236 "TARGET_SIMD"
4237 "ld1\\t{%0<Vmtype>}, %1"
4238 [(set_attr "type" "neon_load1_1reg<q>")]
4239 )
4240
4241 (define_insn "aarch64_be_st1<mode>"
4242 [(set (match_operand:VALLDI 0 "aarch64_simd_struct_operand" "=Utv")
4243 (unspec:VALLDI [(match_operand:VALLDI 1 "register_operand" "w")]
4244 UNSPEC_ST1))]
4245 "TARGET_SIMD"
4246 "st1\\t{%1<Vmtype>}, %0"
4247 [(set_attr "type" "neon_store1_1reg<q>")]
4248 )
4249
4250 (define_split
4251 [(set (match_operand:OI 0 "register_operand" "")
4252 (match_operand:OI 1 "register_operand" ""))]
4253 "TARGET_SIMD && reload_completed"
4254 [(set (match_dup 0) (match_dup 1))
4255 (set (match_dup 2) (match_dup 3))]
4256 {
4257 int rdest = REGNO (operands[0]);
4258 int rsrc = REGNO (operands[1]);
4259 rtx dest[2], src[2];
4260
4261 dest[0] = gen_rtx_REG (TFmode, rdest);
4262 src[0] = gen_rtx_REG (TFmode, rsrc);
4263 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4264 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4265
4266 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
4267 })
4268
4269 (define_split
4270 [(set (match_operand:CI 0 "register_operand" "")
4271 (match_operand:CI 1 "register_operand" ""))]
4272 "TARGET_SIMD && reload_completed"
4273 [(set (match_dup 0) (match_dup 1))
4274 (set (match_dup 2) (match_dup 3))
4275 (set (match_dup 4) (match_dup 5))]
4276 {
4277 int rdest = REGNO (operands[0]);
4278 int rsrc = REGNO (operands[1]);
4279 rtx dest[3], src[3];
4280
4281 dest[0] = gen_rtx_REG (TFmode, rdest);
4282 src[0] = gen_rtx_REG (TFmode, rsrc);
4283 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4284 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4285 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4286 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4287
4288 aarch64_simd_disambiguate_copy (operands, dest, src, 3);
4289 })
4290
4291 (define_split
4292 [(set (match_operand:XI 0 "register_operand" "")
4293 (match_operand:XI 1 "register_operand" ""))]
4294 "TARGET_SIMD && reload_completed"
4295 [(set (match_dup 0) (match_dup 1))
4296 (set (match_dup 2) (match_dup 3))
4297 (set (match_dup 4) (match_dup 5))
4298 (set (match_dup 6) (match_dup 7))]
4299 {
4300 int rdest = REGNO (operands[0]);
4301 int rsrc = REGNO (operands[1]);
4302 rtx dest[4], src[4];
4303
4304 dest[0] = gen_rtx_REG (TFmode, rdest);
4305 src[0] = gen_rtx_REG (TFmode, rsrc);
4306 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4307 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4308 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4309 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4310 dest[3] = gen_rtx_REG (TFmode, rdest + 3);
4311 src[3] = gen_rtx_REG (TFmode, rsrc + 3);
4312
4313 aarch64_simd_disambiguate_copy (operands, dest, src, 4);
4314 })
4315
4316 (define_expand "aarch64_ld2r<mode>"
4317 [(match_operand:OI 0 "register_operand" "=w")
4318 (match_operand:DI 1 "register_operand" "w")
4319 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4320 "TARGET_SIMD"
4321 {
4322 machine_mode mode = <V_TWO_ELEM>mode;
4323 rtx mem = gen_rtx_MEM (mode, operands[1]);
4324
4325 emit_insn (gen_aarch64_simd_ld2r<mode> (operands[0], mem));
4326 DONE;
4327 })
4328
4329 (define_expand "aarch64_ld3r<mode>"
4330 [(match_operand:CI 0 "register_operand" "=w")
4331 (match_operand:DI 1 "register_operand" "w")
4332 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4333 "TARGET_SIMD"
4334 {
4335 machine_mode mode = <V_THREE_ELEM>mode;
4336 rtx mem = gen_rtx_MEM (mode, operands[1]);
4337
4338 emit_insn (gen_aarch64_simd_ld3r<mode> (operands[0], mem));
4339 DONE;
4340 })
4341
4342 (define_expand "aarch64_ld4r<mode>"
4343 [(match_operand:XI 0 "register_operand" "=w")
4344 (match_operand:DI 1 "register_operand" "w")
4345 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4346 "TARGET_SIMD"
4347 {
4348 machine_mode mode = <V_FOUR_ELEM>mode;
4349 rtx mem = gen_rtx_MEM (mode, operands[1]);
4350
4351 emit_insn (gen_aarch64_simd_ld4r<mode> (operands[0],mem));
4352 DONE;
4353 })
4354
4355 (define_insn "aarch64_ld2<mode>_dreg"
4356 [(set (match_operand:OI 0 "register_operand" "=w")
4357 (subreg:OI
4358 (vec_concat:<VRL2>
4359 (vec_concat:<VDBL>
4360 (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4361 UNSPEC_LD2)
4362 (vec_duplicate:VD (const_int 0)))
4363 (vec_concat:<VDBL>
4364 (unspec:VD [(match_dup 1)]
4365 UNSPEC_LD2)
4366 (vec_duplicate:VD (const_int 0)))) 0))]
4367 "TARGET_SIMD"
4368 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4369 [(set_attr "type" "neon_load2_2reg<q>")]
4370 )
4371
4372 (define_insn "aarch64_ld2<mode>_dreg"
4373 [(set (match_operand:OI 0 "register_operand" "=w")
4374 (subreg:OI
4375 (vec_concat:<VRL2>
4376 (vec_concat:<VDBL>
4377 (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4378 UNSPEC_LD2)
4379 (const_int 0))
4380 (vec_concat:<VDBL>
4381 (unspec:DX [(match_dup 1)]
4382 UNSPEC_LD2)
4383 (const_int 0))) 0))]
4384 "TARGET_SIMD"
4385 "ld1\\t{%S0.1d - %T0.1d}, %1"
4386 [(set_attr "type" "neon_load1_2reg<q>")]
4387 )
4388
4389 (define_insn "aarch64_ld3<mode>_dreg"
4390 [(set (match_operand:CI 0 "register_operand" "=w")
4391 (subreg:CI
4392 (vec_concat:<VRL3>
4393 (vec_concat:<VRL2>
4394 (vec_concat:<VDBL>
4395 (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4396 UNSPEC_LD3)
4397 (vec_duplicate:VD (const_int 0)))
4398 (vec_concat:<VDBL>
4399 (unspec:VD [(match_dup 1)]
4400 UNSPEC_LD3)
4401 (vec_duplicate:VD (const_int 0))))
4402 (vec_concat:<VDBL>
4403 (unspec:VD [(match_dup 1)]
4404 UNSPEC_LD3)
4405 (vec_duplicate:VD (const_int 0)))) 0))]
4406 "TARGET_SIMD"
4407 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4408 [(set_attr "type" "neon_load3_3reg<q>")]
4409 )
4410
4411 (define_insn "aarch64_ld3<mode>_dreg"
4412 [(set (match_operand:CI 0 "register_operand" "=w")
4413 (subreg:CI
4414 (vec_concat:<VRL3>
4415 (vec_concat:<VRL2>
4416 (vec_concat:<VDBL>
4417 (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4418 UNSPEC_LD3)
4419 (const_int 0))
4420 (vec_concat:<VDBL>
4421 (unspec:DX [(match_dup 1)]
4422 UNSPEC_LD3)
4423 (const_int 0)))
4424 (vec_concat:<VDBL>
4425 (unspec:DX [(match_dup 1)]
4426 UNSPEC_LD3)
4427 (const_int 0))) 0))]
4428 "TARGET_SIMD"
4429 "ld1\\t{%S0.1d - %U0.1d}, %1"
4430 [(set_attr "type" "neon_load1_3reg<q>")]
4431 )
4432
4433 (define_insn "aarch64_ld4<mode>_dreg"
4434 [(set (match_operand:XI 0 "register_operand" "=w")
4435 (subreg:XI
4436 (vec_concat:<VRL4>
4437 (vec_concat:<VRL2>
4438 (vec_concat:<VDBL>
4439 (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4440 UNSPEC_LD4)
4441 (vec_duplicate:VD (const_int 0)))
4442 (vec_concat:<VDBL>
4443 (unspec:VD [(match_dup 1)]
4444 UNSPEC_LD4)
4445 (vec_duplicate:VD (const_int 0))))
4446 (vec_concat:<VRL2>
4447 (vec_concat:<VDBL>
4448 (unspec:VD [(match_dup 1)]
4449 UNSPEC_LD4)
4450 (vec_duplicate:VD (const_int 0)))
4451 (vec_concat:<VDBL>
4452 (unspec:VD [(match_dup 1)]
4453 UNSPEC_LD4)
4454 (vec_duplicate:VD (const_int 0))))) 0))]
4455 "TARGET_SIMD"
4456 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4457 [(set_attr "type" "neon_load4_4reg<q>")]
4458 )
4459
4460 (define_insn "aarch64_ld4<mode>_dreg"
4461 [(set (match_operand:XI 0 "register_operand" "=w")
4462 (subreg:XI
4463 (vec_concat:<VRL4>
4464 (vec_concat:<VRL2>
4465 (vec_concat:<VDBL>
4466 (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4467 UNSPEC_LD4)
4468 (const_int 0))
4469 (vec_concat:<VDBL>
4470 (unspec:DX [(match_dup 1)]
4471 UNSPEC_LD4)
4472 (const_int 0)))
4473 (vec_concat:<VRL2>
4474 (vec_concat:<VDBL>
4475 (unspec:DX [(match_dup 1)]
4476 UNSPEC_LD4)
4477 (const_int 0))
4478 (vec_concat:<VDBL>
4479 (unspec:DX [(match_dup 1)]
4480 UNSPEC_LD4)
4481 (const_int 0)))) 0))]
4482 "TARGET_SIMD"
4483 "ld1\\t{%S0.1d - %V0.1d}, %1"
4484 [(set_attr "type" "neon_load1_4reg<q>")]
4485 )
4486
4487 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4488 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4489 (match_operand:DI 1 "register_operand" "r")
4490 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4491 "TARGET_SIMD"
4492 {
4493 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4494 rtx mem = gen_rtx_MEM (mode, operands[1]);
4495
4496 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4497 DONE;
4498 })
4499
4500 (define_expand "aarch64_ld1<VALL:mode>"
4501 [(match_operand:VALL 0 "register_operand")
4502 (match_operand:DI 1 "register_operand")]
4503 "TARGET_SIMD"
4504 {
4505 machine_mode mode = <VALL:MODE>mode;
4506 rtx mem = gen_rtx_MEM (mode, operands[1]);
4507
4508 if (BYTES_BIG_ENDIAN)
4509 emit_insn (gen_aarch64_be_ld1<VALL:mode> (operands[0], mem));
4510 else
4511 emit_move_insn (operands[0], mem);
4512 DONE;
4513 })
4514
4515 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4516 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4517 (match_operand:DI 1 "register_operand" "r")
4518 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4519 "TARGET_SIMD"
4520 {
4521 machine_mode mode = <VSTRUCT:MODE>mode;
4522 rtx mem = gen_rtx_MEM (mode, operands[1]);
4523
4524 emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
4525 DONE;
4526 })
4527
4528 (define_expand "aarch64_ld2_lane<mode>"
4529 [(match_operand:OI 0 "register_operand" "=w")
4530 (match_operand:DI 1 "register_operand" "w")
4531 (match_operand:OI 2 "register_operand" "0")
4532 (match_operand:SI 3 "immediate_operand" "i")
4533 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4534 "TARGET_SIMD"
4535 {
4536 machine_mode mode = <V_TWO_ELEM>mode;
4537 rtx mem = gen_rtx_MEM (mode, operands[1]);
4538
4539 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
4540 emit_insn (gen_aarch64_vec_load_lanesoi_lane<mode> (operands[0],
4541 mem,
4542 operands[2],
4543 operands[3]));
4544 DONE;
4545 })
4546
4547 (define_expand "aarch64_ld3_lane<mode>"
4548 [(match_operand:CI 0 "register_operand" "=w")
4549 (match_operand:DI 1 "register_operand" "w")
4550 (match_operand:CI 2 "register_operand" "0")
4551 (match_operand:SI 3 "immediate_operand" "i")
4552 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4553 "TARGET_SIMD"
4554 {
4555 machine_mode mode = <V_THREE_ELEM>mode;
4556 rtx mem = gen_rtx_MEM (mode, operands[1]);
4557
4558 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
4559 emit_insn (gen_aarch64_vec_load_lanesci_lane<mode> (operands[0],
4560 mem,
4561 operands[2],
4562 operands[3]));
4563 DONE;
4564 })
4565
4566 (define_expand "aarch64_ld4_lane<mode>"
4567 [(match_operand:XI 0 "register_operand" "=w")
4568 (match_operand:DI 1 "register_operand" "w")
4569 (match_operand:XI 2 "register_operand" "0")
4570 (match_operand:SI 3 "immediate_operand" "i")
4571 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4572 "TARGET_SIMD"
4573 {
4574 machine_mode mode = <V_FOUR_ELEM>mode;
4575 rtx mem = gen_rtx_MEM (mode, operands[1]);
4576
4577 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
4578 emit_insn (gen_aarch64_vec_load_lanesxi_lane<mode> (operands[0],
4579 mem,
4580 operands[2],
4581 operands[3]));
4582 DONE;
4583 })
4584
4585
4586
4587 ;; Expanders for builtins to extract vector registers from large
4588 ;; opaque integer modes.
4589
4590 ;; D-register list.
4591
4592 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4593 [(match_operand:VDC 0 "register_operand" "=w")
4594 (match_operand:VSTRUCT 1 "register_operand" "w")
4595 (match_operand:SI 2 "immediate_operand" "i")]
4596 "TARGET_SIMD"
4597 {
4598 int part = INTVAL (operands[2]);
4599 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4600 int offset = part * 16;
4601
4602 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4603 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4604 DONE;
4605 })
4606
4607 ;; Q-register list.
4608
4609 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4610 [(match_operand:VQ 0 "register_operand" "=w")
4611 (match_operand:VSTRUCT 1 "register_operand" "w")
4612 (match_operand:SI 2 "immediate_operand" "i")]
4613 "TARGET_SIMD"
4614 {
4615 int part = INTVAL (operands[2]);
4616 int offset = part * 16;
4617
4618 emit_move_insn (operands[0],
4619 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4620 DONE;
4621 })
4622
4623 ;; Permuted-store expanders for neon intrinsics.
4624
4625 ;; Permute instructions
4626
4627 ;; vec_perm support
4628
4629 (define_expand "vec_perm_const<mode>"
4630 [(match_operand:VALL 0 "register_operand")
4631 (match_operand:VALL 1 "register_operand")
4632 (match_operand:VALL 2 "register_operand")
4633 (match_operand:<V_cmp_result> 3)]
4634 "TARGET_SIMD"
4635 {
4636 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4637 operands[2], operands[3]))
4638 DONE;
4639 else
4640 FAIL;
4641 })
4642
4643 (define_expand "vec_perm<mode>"
4644 [(match_operand:VB 0 "register_operand")
4645 (match_operand:VB 1 "register_operand")
4646 (match_operand:VB 2 "register_operand")
4647 (match_operand:VB 3 "register_operand")]
4648 "TARGET_SIMD"
4649 {
4650 aarch64_expand_vec_perm (operands[0], operands[1],
4651 operands[2], operands[3]);
4652 DONE;
4653 })
4654
4655 (define_insn "aarch64_tbl1<mode>"
4656 [(set (match_operand:VB 0 "register_operand" "=w")
4657 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4658 (match_operand:VB 2 "register_operand" "w")]
4659 UNSPEC_TBL))]
4660 "TARGET_SIMD"
4661 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
4662 [(set_attr "type" "neon_tbl1<q>")]
4663 )
4664
4665 ;; Two source registers.
4666
4667 (define_insn "aarch64_tbl2v16qi"
4668 [(set (match_operand:V16QI 0 "register_operand" "=w")
4669 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4670 (match_operand:V16QI 2 "register_operand" "w")]
4671 UNSPEC_TBL))]
4672 "TARGET_SIMD"
4673 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
4674 [(set_attr "type" "neon_tbl2_q")]
4675 )
4676
4677 (define_insn_and_split "aarch64_combinev16qi"
4678 [(set (match_operand:OI 0 "register_operand" "=w")
4679 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4680 (match_operand:V16QI 2 "register_operand" "w")]
4681 UNSPEC_CONCAT))]
4682 "TARGET_SIMD"
4683 "#"
4684 "&& reload_completed"
4685 [(const_int 0)]
4686 {
4687 aarch64_split_combinev16qi (operands);
4688 DONE;
4689 }
4690 [(set_attr "type" "multiple")]
4691 )
4692
4693 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4694 [(set (match_operand:VALL 0 "register_operand" "=w")
4695 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4696 (match_operand:VALL 2 "register_operand" "w")]
4697 PERMUTE))]
4698 "TARGET_SIMD"
4699 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4700 [(set_attr "type" "neon_permute<q>")]
4701 )
4702
4703 ;; Note immediate (third) operand is lane index not byte index.
4704 (define_insn "aarch64_ext<mode>"
4705 [(set (match_operand:VALL 0 "register_operand" "=w")
4706 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4707 (match_operand:VALL 2 "register_operand" "w")
4708 (match_operand:SI 3 "immediate_operand" "i")]
4709 UNSPEC_EXT))]
4710 "TARGET_SIMD"
4711 {
4712 operands[3] = GEN_INT (INTVAL (operands[3])
4713 * GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
4714 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
4715 }
4716 [(set_attr "type" "neon_ext<q>")]
4717 )
4718
4719 ;; This exists solely to check the arguments to the corresponding __builtin.
4720 ;; Used where we want an error for out-of-range indices which would otherwise
4721 ;; be silently wrapped (e.g. the mask to a __builtin_shuffle).
4722 (define_expand "aarch64_im_lane_boundsi"
4723 [(match_operand:SI 0 "immediate_operand" "i")
4724 (match_operand:SI 1 "immediate_operand" "i")]
4725 "TARGET_SIMD"
4726 {
4727 aarch64_simd_lane_bounds (operands[0], 0, INTVAL (operands[1]));
4728 DONE;
4729 }
4730 )
4731
4732 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
4733 [(set (match_operand:VALL 0 "register_operand" "=w")
4734 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
4735 REVERSE))]
4736 "TARGET_SIMD"
4737 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
4738 [(set_attr "type" "neon_rev<q>")]
4739 )
4740
4741 (define_insn "aarch64_st2<mode>_dreg"
4742 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4743 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4744 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4745 UNSPEC_ST2))]
4746 "TARGET_SIMD"
4747 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4748 [(set_attr "type" "neon_store2_2reg")]
4749 )
4750
4751 (define_insn "aarch64_st2<mode>_dreg"
4752 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4753 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4754 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4755 UNSPEC_ST2))]
4756 "TARGET_SIMD"
4757 "st1\\t{%S1.1d - %T1.1d}, %0"
4758 [(set_attr "type" "neon_store1_2reg")]
4759 )
4760
4761 (define_insn "aarch64_st3<mode>_dreg"
4762 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4763 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4764 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4765 UNSPEC_ST3))]
4766 "TARGET_SIMD"
4767 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4768 [(set_attr "type" "neon_store3_3reg")]
4769 )
4770
4771 (define_insn "aarch64_st3<mode>_dreg"
4772 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4773 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4774 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4775 UNSPEC_ST3))]
4776 "TARGET_SIMD"
4777 "st1\\t{%S1.1d - %U1.1d}, %0"
4778 [(set_attr "type" "neon_store1_3reg")]
4779 )
4780
4781 (define_insn "aarch64_st4<mode>_dreg"
4782 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4783 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4784 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4785 UNSPEC_ST4))]
4786 "TARGET_SIMD"
4787 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4788 [(set_attr "type" "neon_store4_4reg")]
4789 )
4790
4791 (define_insn "aarch64_st4<mode>_dreg"
4792 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4793 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4794 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795 UNSPEC_ST4))]
4796 "TARGET_SIMD"
4797 "st1\\t{%S1.1d - %V1.1d}, %0"
4798 [(set_attr "type" "neon_store1_4reg")]
4799 )
4800
4801 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4802 [(match_operand:DI 0 "register_operand" "r")
4803 (match_operand:VSTRUCT 1 "register_operand" "w")
4804 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4805 "TARGET_SIMD"
4806 {
4807 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4808 rtx mem = gen_rtx_MEM (mode, operands[0]);
4809
4810 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4811 DONE;
4812 })
4813
4814 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4815 [(match_operand:DI 0 "register_operand" "r")
4816 (match_operand:VSTRUCT 1 "register_operand" "w")
4817 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4818 "TARGET_SIMD"
4819 {
4820 machine_mode mode = <VSTRUCT:MODE>mode;
4821 rtx mem = gen_rtx_MEM (mode, operands[0]);
4822
4823 emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4824 DONE;
4825 })
4826
4827 (define_expand "aarch64_st2_lane<VQ:mode>"
4828 [(match_operand:DI 0 "register_operand" "r")
4829 (match_operand:OI 1 "register_operand" "w")
4830 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4831 (match_operand:SI 2 "immediate_operand")]
4832 "TARGET_SIMD"
4833 {
4834 machine_mode mode = <V_TWO_ELEM>mode;
4835 rtx mem = gen_rtx_MEM (mode, operands[0]);
4836 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4837
4838 emit_insn (gen_vec_store_lanesoi_lane<VQ:mode> (mem,
4839 operands[1],
4840 operands[2]));
4841 DONE;
4842 })
4843
4844 (define_expand "aarch64_st3_lane<VQ:mode>"
4845 [(match_operand:DI 0 "register_operand" "r")
4846 (match_operand:CI 1 "register_operand" "w")
4847 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4848 (match_operand:SI 2 "immediate_operand")]
4849 "TARGET_SIMD"
4850 {
4851 machine_mode mode = <V_THREE_ELEM>mode;
4852 rtx mem = gen_rtx_MEM (mode, operands[0]);
4853 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4854
4855 emit_insn (gen_vec_store_lanesci_lane<VQ:mode> (mem,
4856 operands[1],
4857 operands[2]));
4858 DONE;
4859 })
4860
4861 (define_expand "aarch64_st4_lane<VQ:mode>"
4862 [(match_operand:DI 0 "register_operand" "r")
4863 (match_operand:XI 1 "register_operand" "w")
4864 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4865 (match_operand:SI 2 "immediate_operand")]
4866 "TARGET_SIMD"
4867 {
4868 machine_mode mode = <V_FOUR_ELEM>mode;
4869 rtx mem = gen_rtx_MEM (mode, operands[0]);
4870 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4871
4872 emit_insn (gen_vec_store_lanesxi_lane<VQ:mode> (mem,
4873 operands[1],
4874 operands[2]));
4875 DONE;
4876 })
4877
4878 (define_expand "aarch64_st1<VALL:mode>"
4879 [(match_operand:DI 0 "register_operand")
4880 (match_operand:VALL 1 "register_operand")]
4881 "TARGET_SIMD"
4882 {
4883 machine_mode mode = <VALL:MODE>mode;
4884 rtx mem = gen_rtx_MEM (mode, operands[0]);
4885
4886 if (BYTES_BIG_ENDIAN)
4887 emit_insn (gen_aarch64_be_st1<VALL:mode> (mem, operands[1]));
4888 else
4889 emit_move_insn (mem, operands[1]);
4890 DONE;
4891 })
4892
4893 ;; Expander for builtins to insert vector registers into large
4894 ;; opaque integer modes.
4895
4896 ;; Q-register list. We don't need a D-reg inserter as we zero
4897 ;; extend them in arm_neon.h and insert the resulting Q-regs.
4898
4899 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4900 [(match_operand:VSTRUCT 0 "register_operand" "+w")
4901 (match_operand:VSTRUCT 1 "register_operand" "0")
4902 (match_operand:VQ 2 "register_operand" "w")
4903 (match_operand:SI 3 "immediate_operand" "i")]
4904 "TARGET_SIMD"
4905 {
4906 int part = INTVAL (operands[3]);
4907 int offset = part * 16;
4908
4909 emit_move_insn (operands[0], operands[1]);
4910 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4911 operands[2]);
4912 DONE;
4913 })
4914
4915 ;; Standard pattern name vec_init<mode>.
4916
4917 (define_expand "vec_init<mode>"
4918 [(match_operand:VALL 0 "register_operand" "")
4919 (match_operand 1 "" "")]
4920 "TARGET_SIMD"
4921 {
4922 aarch64_expand_vector_init (operands[0], operands[1]);
4923 DONE;
4924 })
4925
4926 (define_insn "*aarch64_simd_ld1r<mode>"
4927 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4928 (vec_duplicate:VALLDI
4929 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4930 "TARGET_SIMD"
4931 "ld1r\\t{%0.<Vtype>}, %1"
4932 [(set_attr "type" "neon_load1_all_lanes")]
4933 )
4934
4935 (define_insn "aarch64_frecpe<mode>"
4936 [(set (match_operand:VDQF 0 "register_operand" "=w")
4937 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4938 UNSPEC_FRECPE))]
4939 "TARGET_SIMD"
4940 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4941 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
4942 )
4943
4944 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4945 [(set (match_operand:GPF 0 "register_operand" "=w")
4946 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4947 FRECP))]
4948 "TARGET_SIMD"
4949 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
4950 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
4951 )
4952
4953 (define_insn "aarch64_frecps<mode>"
4954 [(set (match_operand:VALLF 0 "register_operand" "=w")
4955 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4956 (match_operand:VALLF 2 "register_operand" "w")]
4957 UNSPEC_FRECPS))]
4958 "TARGET_SIMD"
4959 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4960 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
4961 )
4962
4963 ;; Standard pattern name vec_extract<mode>.
4964
4965 (define_expand "vec_extract<mode>"
4966 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
4967 (match_operand:VALL 1 "register_operand" "")
4968 (match_operand:SI 2 "immediate_operand" "")]
4969 "TARGET_SIMD"
4970 {
4971 emit_insn
4972 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
4973 DONE;
4974 })
4975
4976 ;; aes
4977
4978 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
4979 [(set (match_operand:V16QI 0 "register_operand" "=w")
4980 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
4981 (match_operand:V16QI 2 "register_operand" "w")]
4982 CRYPTO_AES))]
4983 "TARGET_SIMD && TARGET_CRYPTO"
4984 "aes<aes_op>\\t%0.16b, %2.16b"
4985 [(set_attr "type" "crypto_aese")]
4986 )
4987
4988 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
4989 [(set (match_operand:V16QI 0 "register_operand" "=w")
4990 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
4991 CRYPTO_AESMC))]
4992 "TARGET_SIMD && TARGET_CRYPTO"
4993 "aes<aesmc_op>\\t%0.16b, %1.16b"
4994 [(set_attr "type" "crypto_aesmc")]
4995 )
4996
4997 ;; sha1
4998
4999 (define_insn "aarch64_crypto_sha1hsi"
5000 [(set (match_operand:SI 0 "register_operand" "=w")
5001 (unspec:SI [(match_operand:SI 1
5002 "register_operand" "w")]
5003 UNSPEC_SHA1H))]
5004 "TARGET_SIMD && TARGET_CRYPTO"
5005 "sha1h\\t%s0, %s1"
5006 [(set_attr "type" "crypto_sha1_fast")]
5007 )
5008
5009 (define_insn "aarch64_crypto_sha1su1v4si"
5010 [(set (match_operand:V4SI 0 "register_operand" "=w")
5011 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5012 (match_operand:V4SI 2 "register_operand" "w")]
5013 UNSPEC_SHA1SU1))]
5014 "TARGET_SIMD && TARGET_CRYPTO"
5015 "sha1su1\\t%0.4s, %2.4s"
5016 [(set_attr "type" "crypto_sha1_fast")]
5017 )
5018
5019 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5020 [(set (match_operand:V4SI 0 "register_operand" "=w")
5021 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5022 (match_operand:SI 2 "register_operand" "w")
5023 (match_operand:V4SI 3 "register_operand" "w")]
5024 CRYPTO_SHA1))]
5025 "TARGET_SIMD && TARGET_CRYPTO"
5026 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5027 [(set_attr "type" "crypto_sha1_slow")]
5028 )
5029
5030 (define_insn "aarch64_crypto_sha1su0v4si"
5031 [(set (match_operand:V4SI 0 "register_operand" "=w")
5032 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5033 (match_operand:V4SI 2 "register_operand" "w")
5034 (match_operand:V4SI 3 "register_operand" "w")]
5035 UNSPEC_SHA1SU0))]
5036 "TARGET_SIMD && TARGET_CRYPTO"
5037 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5038 [(set_attr "type" "crypto_sha1_xor")]
5039 )
5040
5041 ;; sha256
5042
5043 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5044 [(set (match_operand:V4SI 0 "register_operand" "=w")
5045 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5046 (match_operand:V4SI 2 "register_operand" "w")
5047 (match_operand:V4SI 3 "register_operand" "w")]
5048 CRYPTO_SHA256))]
5049 "TARGET_SIMD && TARGET_CRYPTO"
5050 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5051 [(set_attr "type" "crypto_sha256_slow")]
5052 )
5053
5054 (define_insn "aarch64_crypto_sha256su0v4si"
5055 [(set (match_operand:V4SI 0 "register_operand" "=w")
5056 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5057 (match_operand:V4SI 2 "register_operand" "w")]
5058 UNSPEC_SHA256SU0))]
5059 "TARGET_SIMD &&TARGET_CRYPTO"
5060 "sha256su0\\t%0.4s, %2.4s"
5061 [(set_attr "type" "crypto_sha256_fast")]
5062 )
5063
5064 (define_insn "aarch64_crypto_sha256su1v4si"
5065 [(set (match_operand:V4SI 0 "register_operand" "=w")
5066 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5067 (match_operand:V4SI 2 "register_operand" "w")
5068 (match_operand:V4SI 3 "register_operand" "w")]
5069 UNSPEC_SHA256SU1))]
5070 "TARGET_SIMD &&TARGET_CRYPTO"
5071 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5072 [(set_attr "type" "crypto_sha256_slow")]
5073 )
5074
5075 ;; pmull
5076
5077 (define_insn "aarch64_crypto_pmulldi"
5078 [(set (match_operand:TI 0 "register_operand" "=w")
5079 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5080 (match_operand:DI 2 "register_operand" "w")]
5081 UNSPEC_PMULL))]
5082 "TARGET_SIMD && TARGET_CRYPTO"
5083 "pmull\\t%0.1q, %1.1d, %2.1d"
5084 [(set_attr "type" "neon_mul_d_long")]
5085 )
5086
5087 (define_insn "aarch64_crypto_pmullv2di"
5088 [(set (match_operand:TI 0 "register_operand" "=w")
5089 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5090 (match_operand:V2DI 2 "register_operand" "w")]
5091 UNSPEC_PMULL2))]
5092 "TARGET_SIMD && TARGET_CRYPTO"
5093 "pmull2\\t%0.1q, %1.2d, %2.2d"
5094 [(set_attr "type" "neon_mul_d_long")]
5095 )
Mirror of https://gcc.gnu.org/git/gcc.git