]> git.ipfire.org Git - thirdparty/gcc.git/blob - gcc/config/aarch64/aarch64-simd.md
projects / thirdparty / gcc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
iterators.md (V_cmp_mixed, [...]): New.
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64-simd.md
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2016 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
20
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL_F16 0 "nonimmediate_operand" "")
23 (match_operand:VALL_F16 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
30
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
43
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ_I
47 (match_operand:<VEL> 1 "register_operand" "r, w")))]
48 "TARGET_SIMD"
49 "@
50 dup\\t%0.<Vtype>, %<vw>1
51 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
52 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
53 )
54
55 (define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
57 (vec_duplicate:VDQF_F16
58 (match_operand:<VEL> 1 "register_operand" "w")))]
59 "TARGET_SIMD"
60 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
61 [(set_attr "type" "neon_dup<q>")]
62 )
63
64 (define_insn "aarch64_dup_lane<mode>"
65 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
66 (vec_duplicate:VALL_F16
67 (vec_select:<VEL>
68 (match_operand:VALL_F16 1 "register_operand" "w")
69 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
70 )))]
71 "TARGET_SIMD"
72 {
73 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
74 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
75 }
76 [(set_attr "type" "neon_dup<q>")]
77 )
78
79 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
80 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
81 (vec_duplicate:VALL_F16
82 (vec_select:<VEL>
83 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
84 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
85 )))]
86 "TARGET_SIMD"
87 {
88 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
89 INTVAL (operands[2])));
90 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
91 }
92 [(set_attr "type" "neon_dup<q>")]
93 )
94
95 (define_insn "*aarch64_simd_mov<mode>"
96 [(set (match_operand:VD 0 "nonimmediate_operand"
97 "=w, m, w, ?r, ?w, ?r, w")
98 (match_operand:VD 1 "general_operand"
99 "m, w, w, w, r, r, Dn"))]
100 "TARGET_SIMD
101 && (register_operand (operands[0], <MODE>mode)
102 || register_operand (operands[1], <MODE>mode))"
103 {
104 switch (which_alternative)
105 {
106 case 0: return "ldr\\t%d0, %1";
107 case 1: return "str\\t%d1, %0";
108 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
109 case 3: return "umov\t%0, %1.d[0]";
110 case 4: return "ins\t%0.d[0], %1";
111 case 5: return "mov\t%0, %1";
112 case 6:
113 return aarch64_output_simd_mov_immediate (operands[1],
114 <MODE>mode, 64);
115 default: gcc_unreachable ();
116 }
117 }
118 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
119 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
120 mov_reg, neon_move<q>")]
121 )
122
123 (define_insn "*aarch64_simd_mov<mode>"
124 [(set (match_operand:VQ 0 "nonimmediate_operand"
125 "=w, m, w, ?r, ?w, ?r, w")
126 (match_operand:VQ 1 "general_operand"
127 "m, w, w, w, r, r, Dn"))]
128 "TARGET_SIMD
129 && (register_operand (operands[0], <MODE>mode)
130 || register_operand (operands[1], <MODE>mode))"
131 {
132 switch (which_alternative)
133 {
134 case 0:
135 return "ldr\\t%q0, %1";
136 case 1:
137 return "str\\t%q1, %0";
138 case 2:
139 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
140 case 3:
141 case 4:
142 case 5:
143 return "#";
144 case 6:
145 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
146 default:
147 gcc_unreachable ();
148 }
149 }
150 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
151 neon_logic<q>, multiple, multiple, multiple,\
152 neon_move<q>")
153 (set_attr "length" "4,4,4,8,8,8,4")]
154 )
155
156 (define_insn "load_pair<mode>"
157 [(set (match_operand:VD 0 "register_operand" "=w")
158 (match_operand:VD 1 "aarch64_mem_pair_operand" "Ump"))
159 (set (match_operand:VD 2 "register_operand" "=w")
160 (match_operand:VD 3 "memory_operand" "m"))]
161 "TARGET_SIMD
162 && rtx_equal_p (XEXP (operands[3], 0),
163 plus_constant (Pmode,
164 XEXP (operands[1], 0),
165 GET_MODE_SIZE (<MODE>mode)))"
166 "ldp\\t%d0, %d2, %1"
167 [(set_attr "type" "neon_ldp")]
168 )
169
170 (define_insn "store_pair<mode>"
171 [(set (match_operand:VD 0 "aarch64_mem_pair_operand" "=Ump")
172 (match_operand:VD 1 "register_operand" "w"))
173 (set (match_operand:VD 2 "memory_operand" "=m")
174 (match_operand:VD 3 "register_operand" "w"))]
175 "TARGET_SIMD
176 && rtx_equal_p (XEXP (operands[2], 0),
177 plus_constant (Pmode,
178 XEXP (operands[0], 0),
179 GET_MODE_SIZE (<MODE>mode)))"
180 "stp\\t%d1, %d3, %0"
181 [(set_attr "type" "neon_stp")]
182 )
183
184 (define_split
185 [(set (match_operand:VQ 0 "register_operand" "")
186 (match_operand:VQ 1 "register_operand" ""))]
187 "TARGET_SIMD && reload_completed
188 && GP_REGNUM_P (REGNO (operands[0]))
189 && GP_REGNUM_P (REGNO (operands[1]))"
190 [(const_int 0)]
191 {
192 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
193 DONE;
194 })
195
196 (define_split
197 [(set (match_operand:VQ 0 "register_operand" "")
198 (match_operand:VQ 1 "register_operand" ""))]
199 "TARGET_SIMD && reload_completed
200 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
201 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
202 [(const_int 0)]
203 {
204 aarch64_split_simd_move (operands[0], operands[1]);
205 DONE;
206 })
207
208 (define_expand "aarch64_split_simd_mov<mode>"
209 [(set (match_operand:VQ 0)
210 (match_operand:VQ 1))]
211 "TARGET_SIMD"
212 {
213 rtx dst = operands[0];
214 rtx src = operands[1];
215
216 if (GP_REGNUM_P (REGNO (src)))
217 {
218 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
219 rtx src_high_part = gen_highpart (<VHALF>mode, src);
220
221 emit_insn
222 (gen_move_lo_quad_<mode> (dst, src_low_part));
223 emit_insn
224 (gen_move_hi_quad_<mode> (dst, src_high_part));
225 }
226
227 else
228 {
229 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
230 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
231 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
232 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
233
234 emit_insn
235 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
236 emit_insn
237 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
238 }
239 DONE;
240 }
241 )
242
243 (define_insn "aarch64_simd_mov_from_<mode>low"
244 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
245 (vec_select:<VHALF>
246 (match_operand:VQ 1 "register_operand" "w")
247 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
248 "TARGET_SIMD && reload_completed"
249 "umov\t%0, %1.d[0]"
250 [(set_attr "type" "neon_to_gp<q>")
251 (set_attr "length" "4")
252 ])
253
254 (define_insn "aarch64_simd_mov_from_<mode>high"
255 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
256 (vec_select:<VHALF>
257 (match_operand:VQ 1 "register_operand" "w")
258 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
259 "TARGET_SIMD && reload_completed"
260 "umov\t%0, %1.d[1]"
261 [(set_attr "type" "neon_to_gp<q>")
262 (set_attr "length" "4")
263 ])
264
265 (define_insn "orn<mode>3"
266 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
267 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
268 (match_operand:VDQ_I 2 "register_operand" "w")))]
269 "TARGET_SIMD"
270 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
271 [(set_attr "type" "neon_logic<q>")]
272 )
273
274 (define_insn "bic<mode>3"
275 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
276 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
277 (match_operand:VDQ_I 2 "register_operand" "w")))]
278 "TARGET_SIMD"
279 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
280 [(set_attr "type" "neon_logic<q>")]
281 )
282
283 (define_insn "add<mode>3"
284 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
285 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
286 (match_operand:VDQ_I 2 "register_operand" "w")))]
287 "TARGET_SIMD"
288 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
289 [(set_attr "type" "neon_add<q>")]
290 )
291
292 (define_insn "sub<mode>3"
293 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
294 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
295 (match_operand:VDQ_I 2 "register_operand" "w")))]
296 "TARGET_SIMD"
297 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
298 [(set_attr "type" "neon_sub<q>")]
299 )
300
301 (define_insn "mul<mode>3"
302 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
303 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
304 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
305 "TARGET_SIMD"
306 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
307 [(set_attr "type" "neon_mul_<Vetype><q>")]
308 )
309
310 (define_insn "bswap<mode>2"
311 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
312 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
313 "TARGET_SIMD"
314 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
315 [(set_attr "type" "neon_rev<q>")]
316 )
317
318 (define_insn "aarch64_rbit<mode>"
319 [(set (match_operand:VB 0 "register_operand" "=w")
320 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
321 UNSPEC_RBIT))]
322 "TARGET_SIMD"
323 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
324 [(set_attr "type" "neon_rbit")]
325 )
326
327 (define_expand "ctz<mode>2"
328 [(set (match_operand:VS 0 "register_operand")
329 (ctz:VS (match_operand:VS 1 "register_operand")))]
330 "TARGET_SIMD"
331 {
332 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
333 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
334 <MODE>mode, 0);
335 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
336 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
337 DONE;
338 }
339 )
340
341 (define_insn "*aarch64_mul3_elt<mode>"
342 [(set (match_operand:VMUL 0 "register_operand" "=w")
343 (mult:VMUL
344 (vec_duplicate:VMUL
345 (vec_select:<VEL>
346 (match_operand:VMUL 1 "register_operand" "<h_con>")
347 (parallel [(match_operand:SI 2 "immediate_operand")])))
348 (match_operand:VMUL 3 "register_operand" "w")))]
349 "TARGET_SIMD"
350 {
351 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
352 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
353 }
354 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
355 )
356
357 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
358 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
359 (mult:VMUL_CHANGE_NLANES
360 (vec_duplicate:VMUL_CHANGE_NLANES
361 (vec_select:<VEL>
362 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
363 (parallel [(match_operand:SI 2 "immediate_operand")])))
364 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
365 "TARGET_SIMD"
366 {
367 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
368 INTVAL (operands[2])));
369 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
370 }
371 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
372 )
373
374 (define_insn "*aarch64_mul3_elt_from_dup<mode>"
375 [(set (match_operand:VMUL 0 "register_operand" "=w")
376 (mult:VMUL
377 (vec_duplicate:VMUL
378 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
379 (match_operand:VMUL 2 "register_operand" "w")))]
380 "TARGET_SIMD"
381 "<f>mul\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]";
382 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
383 )
384
385 (define_insn "aarch64_rsqrte<mode>"
386 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
387 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")]
388 UNSPEC_RSQRTE))]
389 "TARGET_SIMD"
390 "frsqrte\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
391 [(set_attr "type" "neon_fp_rsqrte_<stype><q>")])
392
393 (define_insn "aarch64_rsqrts<mode>"
394 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
395 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
396 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
397 UNSPEC_RSQRTS))]
398 "TARGET_SIMD"
399 "frsqrts\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
400 [(set_attr "type" "neon_fp_rsqrts_<stype><q>")])
401
402 (define_expand "rsqrt<mode>2"
403 [(set (match_operand:VALLF 0 "register_operand" "=w")
404 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
405 UNSPEC_RSQRT))]
406 "TARGET_SIMD"
407 {
408 aarch64_emit_approx_sqrt (operands[0], operands[1], true);
409 DONE;
410 })
411
412 (define_insn "*aarch64_mul3_elt_to_64v2df"
413 [(set (match_operand:DF 0 "register_operand" "=w")
414 (mult:DF
415 (vec_select:DF
416 (match_operand:V2DF 1 "register_operand" "w")
417 (parallel [(match_operand:SI 2 "immediate_operand")]))
418 (match_operand:DF 3 "register_operand" "w")))]
419 "TARGET_SIMD"
420 {
421 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
422 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
423 }
424 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
425 )
426
427 (define_insn "neg<mode>2"
428 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
429 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
430 "TARGET_SIMD"
431 "neg\t%0.<Vtype>, %1.<Vtype>"
432 [(set_attr "type" "neon_neg<q>")]
433 )
434
435 (define_insn "abs<mode>2"
436 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
437 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
438 "TARGET_SIMD"
439 "abs\t%0.<Vtype>, %1.<Vtype>"
440 [(set_attr "type" "neon_abs<q>")]
441 )
442
443 ;; The intrinsic version of integer ABS must not be allowed to
444 ;; combine with any operation with an integerated ABS step, such
445 ;; as SABD.
446 (define_insn "aarch64_abs<mode>"
447 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
448 (unspec:VSDQ_I_DI
449 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")]
450 UNSPEC_ABS))]
451 "TARGET_SIMD"
452 "abs\t%<v>0<Vmtype>, %<v>1<Vmtype>"
453 [(set_attr "type" "neon_abs<q>")]
454 )
455
456 (define_insn "abd<mode>_3"
457 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
458 (abs:VDQ_BHSI (minus:VDQ_BHSI
459 (match_operand:VDQ_BHSI 1 "register_operand" "w")
460 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
461 "TARGET_SIMD"
462 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
463 [(set_attr "type" "neon_abd<q>")]
464 )
465
466 (define_insn "aba<mode>_3"
467 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
468 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
469 (match_operand:VDQ_BHSI 1 "register_operand" "w")
470 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
471 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
472 "TARGET_SIMD"
473 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
474 [(set_attr "type" "neon_arith_acc<q>")]
475 )
476
477 (define_insn "fabd<mode>3"
478 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
479 (abs:VHSDF_HSDF
480 (minus:VHSDF_HSDF
481 (match_operand:VHSDF_HSDF 1 "register_operand" "w")
482 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))))]
483 "TARGET_SIMD"
484 "fabd\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
485 [(set_attr "type" "neon_fp_abd_<stype><q>")]
486 )
487
488 (define_insn "and<mode>3"
489 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
490 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
491 (match_operand:VDQ_I 2 "register_operand" "w")))]
492 "TARGET_SIMD"
493 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
494 [(set_attr "type" "neon_logic<q>")]
495 )
496
497 (define_insn "ior<mode>3"
498 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
499 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
500 (match_operand:VDQ_I 2 "register_operand" "w")))]
501 "TARGET_SIMD"
502 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
503 [(set_attr "type" "neon_logic<q>")]
504 )
505
506 (define_insn "xor<mode>3"
507 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
508 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
509 (match_operand:VDQ_I 2 "register_operand" "w")))]
510 "TARGET_SIMD"
511 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
512 [(set_attr "type" "neon_logic<q>")]
513 )
514
515 (define_insn "one_cmpl<mode>2"
516 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
517 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
518 "TARGET_SIMD"
519 "not\t%0.<Vbtype>, %1.<Vbtype>"
520 [(set_attr "type" "neon_logic<q>")]
521 )
522
523 (define_insn "aarch64_simd_vec_set<mode>"
524 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
525 (vec_merge:VDQ_BHSI
526 (vec_duplicate:VDQ_BHSI
527 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
528 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
529 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
530 "TARGET_SIMD"
531 {
532 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
533 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
534 switch (which_alternative)
535 {
536 case 0:
537 return "ins\\t%0.<Vetype>[%p2], %w1";
538 case 1:
539 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
540 case 2:
541 return "ld1\\t{%0.<Vetype>}[%p2], %1";
542 default:
543 gcc_unreachable ();
544 }
545 }
546 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_1reg<q>")]
547 )
548
549 (define_insn "*aarch64_simd_vec_copy_lane<mode>"
550 [(set (match_operand:VALL 0 "register_operand" "=w")
551 (vec_merge:VALL
552 (vec_duplicate:VALL
553 (vec_select:<VEL>
554 (match_operand:VALL 3 "register_operand" "w")
555 (parallel
556 [(match_operand:SI 4 "immediate_operand" "i")])))
557 (match_operand:VALL 1 "register_operand" "0")
558 (match_operand:SI 2 "immediate_operand" "i")))]
559 "TARGET_SIMD"
560 {
561 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
562 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
563 operands[4] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[4])));
564
565 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
566 }
567 [(set_attr "type" "neon_ins<q>")]
568 )
569
570 (define_insn "*aarch64_simd_vec_copy_lane_<vswap_width_name><mode>"
571 [(set (match_operand:VALL 0 "register_operand" "=w")
572 (vec_merge:VALL
573 (vec_duplicate:VALL
574 (vec_select:<VEL>
575 (match_operand:<VSWAP_WIDTH> 3 "register_operand" "w")
576 (parallel
577 [(match_operand:SI 4 "immediate_operand" "i")])))
578 (match_operand:VALL 1 "register_operand" "0")
579 (match_operand:SI 2 "immediate_operand" "i")))]
580 "TARGET_SIMD"
581 {
582 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
583 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
584 operands[4] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
585 INTVAL (operands[4])));
586
587 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
588 }
589 [(set_attr "type" "neon_ins<q>")]
590 )
591
592 (define_insn "aarch64_simd_lshr<mode>"
593 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
594 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
595 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
596 "TARGET_SIMD"
597 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
598 [(set_attr "type" "neon_shift_imm<q>")]
599 )
600
601 (define_insn "aarch64_simd_ashr<mode>"
602 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
603 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
604 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
605 "TARGET_SIMD"
606 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
607 [(set_attr "type" "neon_shift_imm<q>")]
608 )
609
610 (define_insn "aarch64_simd_imm_shl<mode>"
611 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
612 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
613 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
614 "TARGET_SIMD"
615 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
616 [(set_attr "type" "neon_shift_imm<q>")]
617 )
618
619 (define_insn "aarch64_simd_reg_sshl<mode>"
620 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
621 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
622 (match_operand:VDQ_I 2 "register_operand" "w")))]
623 "TARGET_SIMD"
624 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
625 [(set_attr "type" "neon_shift_reg<q>")]
626 )
627
628 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
629 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
630 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
631 (match_operand:VDQ_I 2 "register_operand" "w")]
632 UNSPEC_ASHIFT_UNSIGNED))]
633 "TARGET_SIMD"
634 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
635 [(set_attr "type" "neon_shift_reg<q>")]
636 )
637
638 (define_insn "aarch64_simd_reg_shl<mode>_signed"
639 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
640 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
641 (match_operand:VDQ_I 2 "register_operand" "w")]
642 UNSPEC_ASHIFT_SIGNED))]
643 "TARGET_SIMD"
644 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
645 [(set_attr "type" "neon_shift_reg<q>")]
646 )
647
648 (define_expand "ashl<mode>3"
649 [(match_operand:VDQ_I 0 "register_operand" "")
650 (match_operand:VDQ_I 1 "register_operand" "")
651 (match_operand:SI 2 "general_operand" "")]
652 "TARGET_SIMD"
653 {
654 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
655 int shift_amount;
656
657 if (CONST_INT_P (operands[2]))
658 {
659 shift_amount = INTVAL (operands[2]);
660 if (shift_amount >= 0 && shift_amount < bit_width)
661 {
662 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
663 shift_amount);
664 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
665 operands[1],
666 tmp));
667 DONE;
668 }
669 else
670 {
671 operands[2] = force_reg (SImode, operands[2]);
672 }
673 }
674 else if (MEM_P (operands[2]))
675 {
676 operands[2] = force_reg (SImode, operands[2]);
677 }
678
679 if (REG_P (operands[2]))
680 {
681 rtx tmp = gen_reg_rtx (<MODE>mode);
682 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
683 convert_to_mode (<VEL>mode,
684 operands[2],
685 0)));
686 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
687 tmp));
688 DONE;
689 }
690 else
691 FAIL;
692 }
693 )
694
695 (define_expand "lshr<mode>3"
696 [(match_operand:VDQ_I 0 "register_operand" "")
697 (match_operand:VDQ_I 1 "register_operand" "")
698 (match_operand:SI 2 "general_operand" "")]
699 "TARGET_SIMD"
700 {
701 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
702 int shift_amount;
703
704 if (CONST_INT_P (operands[2]))
705 {
706 shift_amount = INTVAL (operands[2]);
707 if (shift_amount > 0 && shift_amount <= bit_width)
708 {
709 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
710 shift_amount);
711 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
712 operands[1],
713 tmp));
714 DONE;
715 }
716 else
717 operands[2] = force_reg (SImode, operands[2]);
718 }
719 else if (MEM_P (operands[2]))
720 {
721 operands[2] = force_reg (SImode, operands[2]);
722 }
723
724 if (REG_P (operands[2]))
725 {
726 rtx tmp = gen_reg_rtx (SImode);
727 rtx tmp1 = gen_reg_rtx (<MODE>mode);
728 emit_insn (gen_negsi2 (tmp, operands[2]));
729 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
730 convert_to_mode (<VEL>mode,
731 tmp, 0)));
732 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
733 operands[1],
734 tmp1));
735 DONE;
736 }
737 else
738 FAIL;
739 }
740 )
741
742 (define_expand "ashr<mode>3"
743 [(match_operand:VDQ_I 0 "register_operand" "")
744 (match_operand:VDQ_I 1 "register_operand" "")
745 (match_operand:SI 2 "general_operand" "")]
746 "TARGET_SIMD"
747 {
748 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
749 int shift_amount;
750
751 if (CONST_INT_P (operands[2]))
752 {
753 shift_amount = INTVAL (operands[2]);
754 if (shift_amount > 0 && shift_amount <= bit_width)
755 {
756 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
757 shift_amount);
758 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
759 operands[1],
760 tmp));
761 DONE;
762 }
763 else
764 operands[2] = force_reg (SImode, operands[2]);
765 }
766 else if (MEM_P (operands[2]))
767 {
768 operands[2] = force_reg (SImode, operands[2]);
769 }
770
771 if (REG_P (operands[2]))
772 {
773 rtx tmp = gen_reg_rtx (SImode);
774 rtx tmp1 = gen_reg_rtx (<MODE>mode);
775 emit_insn (gen_negsi2 (tmp, operands[2]));
776 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
777 convert_to_mode (<VEL>mode,
778 tmp, 0)));
779 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
780 operands[1],
781 tmp1));
782 DONE;
783 }
784 else
785 FAIL;
786 }
787 )
788
789 (define_expand "vashl<mode>3"
790 [(match_operand:VDQ_I 0 "register_operand" "")
791 (match_operand:VDQ_I 1 "register_operand" "")
792 (match_operand:VDQ_I 2 "register_operand" "")]
793 "TARGET_SIMD"
794 {
795 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
796 operands[2]));
797 DONE;
798 })
799
800 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
801 ;; Negating individual lanes most certainly offsets the
802 ;; gain from vectorization.
803 (define_expand "vashr<mode>3"
804 [(match_operand:VDQ_BHSI 0 "register_operand" "")
805 (match_operand:VDQ_BHSI 1 "register_operand" "")
806 (match_operand:VDQ_BHSI 2 "register_operand" "")]
807 "TARGET_SIMD"
808 {
809 rtx neg = gen_reg_rtx (<MODE>mode);
810 emit (gen_neg<mode>2 (neg, operands[2]));
811 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
812 neg));
813 DONE;
814 })
815
816 ;; DI vector shift
817 (define_expand "aarch64_ashr_simddi"
818 [(match_operand:DI 0 "register_operand" "=w")
819 (match_operand:DI 1 "register_operand" "w")
820 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
821 "TARGET_SIMD"
822 {
823 /* An arithmetic shift right by 64 fills the result with copies of the sign
824 bit, just like asr by 63 - however the standard pattern does not handle
825 a shift by 64. */
826 if (INTVAL (operands[2]) == 64)
827 operands[2] = GEN_INT (63);
828 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
829 DONE;
830 }
831 )
832
833 (define_expand "vlshr<mode>3"
834 [(match_operand:VDQ_BHSI 0 "register_operand" "")
835 (match_operand:VDQ_BHSI 1 "register_operand" "")
836 (match_operand:VDQ_BHSI 2 "register_operand" "")]
837 "TARGET_SIMD"
838 {
839 rtx neg = gen_reg_rtx (<MODE>mode);
840 emit (gen_neg<mode>2 (neg, operands[2]));
841 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
842 neg));
843 DONE;
844 })
845
846 (define_expand "aarch64_lshr_simddi"
847 [(match_operand:DI 0 "register_operand" "=w")
848 (match_operand:DI 1 "register_operand" "w")
849 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
850 "TARGET_SIMD"
851 {
852 if (INTVAL (operands[2]) == 64)
853 emit_move_insn (operands[0], const0_rtx);
854 else
855 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
856 DONE;
857 }
858 )
859
860 (define_expand "vec_set<mode>"
861 [(match_operand:VDQ_BHSI 0 "register_operand")
862 (match_operand:<VEL> 1 "register_operand")
863 (match_operand:SI 2 "immediate_operand")]
864 "TARGET_SIMD"
865 {
866 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
867 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
868 GEN_INT (elem), operands[0]));
869 DONE;
870 }
871 )
872
873 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
874 (define_insn "vec_shr_<mode>"
875 [(set (match_operand:VD 0 "register_operand" "=w")
876 (unspec:VD [(match_operand:VD 1 "register_operand" "w")
877 (match_operand:SI 2 "immediate_operand" "i")]
878 UNSPEC_VEC_SHR))]
879 "TARGET_SIMD"
880 {
881 if (BYTES_BIG_ENDIAN)
882 return "shl %d0, %d1, %2";
883 else
884 return "ushr %d0, %d1, %2";
885 }
886 [(set_attr "type" "neon_shift_imm")]
887 )
888
889 (define_insn "aarch64_simd_vec_setv2di"
890 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
891 (vec_merge:V2DI
892 (vec_duplicate:V2DI
893 (match_operand:DI 1 "register_operand" "r,w"))
894 (match_operand:V2DI 3 "register_operand" "0,0")
895 (match_operand:SI 2 "immediate_operand" "i,i")))]
896 "TARGET_SIMD"
897 {
898 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
899 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
900 switch (which_alternative)
901 {
902 case 0:
903 return "ins\\t%0.d[%p2], %1";
904 case 1:
905 return "ins\\t%0.d[%p2], %1.d[0]";
906 default:
907 gcc_unreachable ();
908 }
909 }
910 [(set_attr "type" "neon_from_gp, neon_ins_q")]
911 )
912
913 (define_expand "vec_setv2di"
914 [(match_operand:V2DI 0 "register_operand")
915 (match_operand:DI 1 "register_operand")
916 (match_operand:SI 2 "immediate_operand")]
917 "TARGET_SIMD"
918 {
919 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
920 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
921 GEN_INT (elem), operands[0]));
922 DONE;
923 }
924 )
925
926 (define_insn "aarch64_simd_vec_set<mode>"
927 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
928 (vec_merge:VDQF_F16
929 (vec_duplicate:VDQF_F16
930 (match_operand:<VEL> 1 "register_operand" "w"))
931 (match_operand:VDQF_F16 3 "register_operand" "0")
932 (match_operand:SI 2 "immediate_operand" "i")))]
933 "TARGET_SIMD"
934 {
935 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
936
937 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
938 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
939 }
940 [(set_attr "type" "neon_ins<q>")]
941 )
942
943 (define_expand "vec_set<mode>"
944 [(match_operand:VDQF_F16 0 "register_operand" "+w")
945 (match_operand:<VEL> 1 "register_operand" "w")
946 (match_operand:SI 2 "immediate_operand" "")]
947 "TARGET_SIMD"
948 {
949 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
950 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
951 GEN_INT (elem), operands[0]));
952 DONE;
953 }
954 )
955
956
957 (define_insn "aarch64_mla<mode>"
958 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
959 (plus:VDQ_BHSI (mult:VDQ_BHSI
960 (match_operand:VDQ_BHSI 2 "register_operand" "w")
961 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
962 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
963 "TARGET_SIMD"
964 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
965 [(set_attr "type" "neon_mla_<Vetype><q>")]
966 )
967
968 (define_insn "*aarch64_mla_elt<mode>"
969 [(set (match_operand:VDQHS 0 "register_operand" "=w")
970 (plus:VDQHS
971 (mult:VDQHS
972 (vec_duplicate:VDQHS
973 (vec_select:<VEL>
974 (match_operand:VDQHS 1 "register_operand" "<h_con>")
975 (parallel [(match_operand:SI 2 "immediate_operand")])))
976 (match_operand:VDQHS 3 "register_operand" "w"))
977 (match_operand:VDQHS 4 "register_operand" "0")))]
978 "TARGET_SIMD"
979 {
980 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
981 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
982 }
983 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
984 )
985
986 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
987 [(set (match_operand:VDQHS 0 "register_operand" "=w")
988 (plus:VDQHS
989 (mult:VDQHS
990 (vec_duplicate:VDQHS
991 (vec_select:<VEL>
992 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
993 (parallel [(match_operand:SI 2 "immediate_operand")])))
994 (match_operand:VDQHS 3 "register_operand" "w"))
995 (match_operand:VDQHS 4 "register_operand" "0")))]
996 "TARGET_SIMD"
997 {
998 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
999 INTVAL (operands[2])));
1000 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1001 }
1002 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1003 )
1004
1005 (define_insn "aarch64_mls<mode>"
1006 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1007 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
1008 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
1009 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
1010 "TARGET_SIMD"
1011 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1012 [(set_attr "type" "neon_mla_<Vetype><q>")]
1013 )
1014
1015 (define_insn "*aarch64_mls_elt<mode>"
1016 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1017 (minus:VDQHS
1018 (match_operand:VDQHS 4 "register_operand" "0")
1019 (mult:VDQHS
1020 (vec_duplicate:VDQHS
1021 (vec_select:<VEL>
1022 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1023 (parallel [(match_operand:SI 2 "immediate_operand")])))
1024 (match_operand:VDQHS 3 "register_operand" "w"))))]
1025 "TARGET_SIMD"
1026 {
1027 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1028 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1029 }
1030 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1031 )
1032
1033 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
1034 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1035 (minus:VDQHS
1036 (match_operand:VDQHS 4 "register_operand" "0")
1037 (mult:VDQHS
1038 (vec_duplicate:VDQHS
1039 (vec_select:<VEL>
1040 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1041 (parallel [(match_operand:SI 2 "immediate_operand")])))
1042 (match_operand:VDQHS 3 "register_operand" "w"))))]
1043 "TARGET_SIMD"
1044 {
1045 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1046 INTVAL (operands[2])));
1047 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1048 }
1049 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1050 )
1051
1052 ;; Max/Min operations.
1053 (define_insn "<su><maxmin><mode>3"
1054 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1055 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
1056 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
1057 "TARGET_SIMD"
1058 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1059 [(set_attr "type" "neon_minmax<q>")]
1060 )
1061
1062 (define_expand "<su><maxmin>v2di3"
1063 [(set (match_operand:V2DI 0 "register_operand" "")
1064 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
1065 (match_operand:V2DI 2 "register_operand" "")))]
1066 "TARGET_SIMD"
1067 {
1068 enum rtx_code cmp_operator;
1069 rtx cmp_fmt;
1070
1071 switch (<CODE>)
1072 {
1073 case UMIN:
1074 cmp_operator = LTU;
1075 break;
1076 case SMIN:
1077 cmp_operator = LT;
1078 break;
1079 case UMAX:
1080 cmp_operator = GTU;
1081 break;
1082 case SMAX:
1083 cmp_operator = GT;
1084 break;
1085 default:
1086 gcc_unreachable ();
1087 }
1088
1089 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1090 emit_insn (gen_vcondv2div2di (operands[0], operands[1],
1091 operands[2], cmp_fmt, operands[1], operands[2]));
1092 DONE;
1093 })
1094
1095 ;; Pairwise Integer Max/Min operations.
1096 (define_insn "aarch64_<maxmin_uns>p<mode>"
1097 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1098 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
1099 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
1100 MAXMINV))]
1101 "TARGET_SIMD"
1102 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1103 [(set_attr "type" "neon_minmax<q>")]
1104 )
1105
1106 ;; Pairwise FP Max/Min operations.
1107 (define_insn "aarch64_<maxmin_uns>p<mode>"
1108 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1109 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
1110 (match_operand:VHSDF 2 "register_operand" "w")]
1111 FMAXMINV))]
1112 "TARGET_SIMD"
1113 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1114 [(set_attr "type" "neon_minmax<q>")]
1115 )
1116
1117 ;; vec_concat gives a new vector with the low elements from operand 1, and
1118 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1119 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1120 ;; What that means, is that the RTL descriptions of the below patterns
1121 ;; need to change depending on endianness.
1122
1123 ;; Move to the low architectural bits of the register.
1124 ;; On little-endian this is { operand, zeroes }
1125 ;; On big-endian this is { zeroes, operand }
1126
1127 (define_insn "move_lo_quad_internal_<mode>"
1128 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1129 (vec_concat:VQ_NO2E
1130 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1131 (vec_duplicate:<VHALF> (const_int 0))))]
1132 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1133 "@
1134 dup\\t%d0, %1.d[0]
1135 fmov\\t%d0, %1
1136 dup\\t%d0, %1"
1137 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1138 (set_attr "simd" "yes,*,yes")
1139 (set_attr "fp" "*,yes,*")
1140 (set_attr "length" "4")]
1141 )
1142
1143 (define_insn "move_lo_quad_internal_<mode>"
1144 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1145 (vec_concat:VQ_2E
1146 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1147 (const_int 0)))]
1148 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1149 "@
1150 dup\\t%d0, %1.d[0]
1151 fmov\\t%d0, %1
1152 dup\\t%d0, %1"
1153 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1154 (set_attr "simd" "yes,*,yes")
1155 (set_attr "fp" "*,yes,*")
1156 (set_attr "length" "4")]
1157 )
1158
1159 (define_insn "move_lo_quad_internal_be_<mode>"
1160 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1161 (vec_concat:VQ_NO2E
1162 (vec_duplicate:<VHALF> (const_int 0))
1163 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1164 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1165 "@
1166 dup\\t%d0, %1.d[0]
1167 fmov\\t%d0, %1
1168 dup\\t%d0, %1"
1169 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1170 (set_attr "simd" "yes,*,yes")
1171 (set_attr "fp" "*,yes,*")
1172 (set_attr "length" "4")]
1173 )
1174
1175 (define_insn "move_lo_quad_internal_be_<mode>"
1176 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1177 (vec_concat:VQ_2E
1178 (const_int 0)
1179 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1180 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1181 "@
1182 dup\\t%d0, %1.d[0]
1183 fmov\\t%d0, %1
1184 dup\\t%d0, %1"
1185 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1186 (set_attr "simd" "yes,*,yes")
1187 (set_attr "fp" "*,yes,*")
1188 (set_attr "length" "4")]
1189 )
1190
1191 (define_expand "move_lo_quad_<mode>"
1192 [(match_operand:VQ 0 "register_operand")
1193 (match_operand:VQ 1 "register_operand")]
1194 "TARGET_SIMD"
1195 {
1196 if (BYTES_BIG_ENDIAN)
1197 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1198 else
1199 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1200 DONE;
1201 }
1202 )
1203
1204 ;; Move operand1 to the high architectural bits of the register, keeping
1205 ;; the low architectural bits of operand2.
1206 ;; For little-endian this is { operand2, operand1 }
1207 ;; For big-endian this is { operand1, operand2 }
1208
1209 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1210 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1211 (vec_concat:VQ
1212 (vec_select:<VHALF>
1213 (match_dup 0)
1214 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1215 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1216 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1217 "@
1218 ins\\t%0.d[1], %1.d[0]
1219 ins\\t%0.d[1], %1"
1220 [(set_attr "type" "neon_ins")]
1221 )
1222
1223 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1224 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1225 (vec_concat:VQ
1226 (match_operand:<VHALF> 1 "register_operand" "w,r")
1227 (vec_select:<VHALF>
1228 (match_dup 0)
1229 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1230 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1231 "@
1232 ins\\t%0.d[1], %1.d[0]
1233 ins\\t%0.d[1], %1"
1234 [(set_attr "type" "neon_ins")]
1235 )
1236
1237 (define_expand "move_hi_quad_<mode>"
1238 [(match_operand:VQ 0 "register_operand" "")
1239 (match_operand:<VHALF> 1 "register_operand" "")]
1240 "TARGET_SIMD"
1241 {
1242 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1243 if (BYTES_BIG_ENDIAN)
1244 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1245 operands[1], p));
1246 else
1247 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1248 operands[1], p));
1249 DONE;
1250 })
1251
1252 ;; Narrowing operations.
1253
1254 ;; For doubles.
1255 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1256 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1257 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1258 "TARGET_SIMD"
1259 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1260 [(set_attr "type" "neon_shift_imm_narrow_q")]
1261 )
1262
1263 (define_expand "vec_pack_trunc_<mode>"
1264 [(match_operand:<VNARROWD> 0 "register_operand" "")
1265 (match_operand:VDN 1 "register_operand" "")
1266 (match_operand:VDN 2 "register_operand" "")]
1267 "TARGET_SIMD"
1268 {
1269 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1270 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1271 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1272
1273 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1274 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1275 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1276 DONE;
1277 })
1278
1279 ;; For quads.
1280
1281 (define_insn "vec_pack_trunc_<mode>"
1282 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1283 (vec_concat:<VNARROWQ2>
1284 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1285 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1286 "TARGET_SIMD"
1287 {
1288 if (BYTES_BIG_ENDIAN)
1289 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1290 else
1291 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1292 }
1293 [(set_attr "type" "multiple")
1294 (set_attr "length" "8")]
1295 )
1296
1297 ;; Widening operations.
1298
1299 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1300 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1301 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1302 (match_operand:VQW 1 "register_operand" "w")
1303 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1304 )))]
1305 "TARGET_SIMD"
1306 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1307 [(set_attr "type" "neon_shift_imm_long")]
1308 )
1309
1310 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1311 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1312 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1313 (match_operand:VQW 1 "register_operand" "w")
1314 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1315 )))]
1316 "TARGET_SIMD"
1317 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1318 [(set_attr "type" "neon_shift_imm_long")]
1319 )
1320
1321 (define_expand "vec_unpack<su>_hi_<mode>"
1322 [(match_operand:<VWIDE> 0 "register_operand" "")
1323 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1324 "TARGET_SIMD"
1325 {
1326 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1327 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1328 operands[1], p));
1329 DONE;
1330 }
1331 )
1332
1333 (define_expand "vec_unpack<su>_lo_<mode>"
1334 [(match_operand:<VWIDE> 0 "register_operand" "")
1335 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1336 "TARGET_SIMD"
1337 {
1338 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1339 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1340 operands[1], p));
1341 DONE;
1342 }
1343 )
1344
1345 ;; Widening arithmetic.
1346
1347 (define_insn "*aarch64_<su>mlal_lo<mode>"
1348 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1349 (plus:<VWIDE>
1350 (mult:<VWIDE>
1351 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1352 (match_operand:VQW 2 "register_operand" "w")
1353 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1354 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1355 (match_operand:VQW 4 "register_operand" "w")
1356 (match_dup 3))))
1357 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1358 "TARGET_SIMD"
1359 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1360 [(set_attr "type" "neon_mla_<Vetype>_long")]
1361 )
1362
1363 (define_insn "*aarch64_<su>mlal_hi<mode>"
1364 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1365 (plus:<VWIDE>
1366 (mult:<VWIDE>
1367 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1368 (match_operand:VQW 2 "register_operand" "w")
1369 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1370 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1371 (match_operand:VQW 4 "register_operand" "w")
1372 (match_dup 3))))
1373 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1374 "TARGET_SIMD"
1375 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1376 [(set_attr "type" "neon_mla_<Vetype>_long")]
1377 )
1378
1379 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1380 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1381 (minus:<VWIDE>
1382 (match_operand:<VWIDE> 1 "register_operand" "0")
1383 (mult:<VWIDE>
1384 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1385 (match_operand:VQW 2 "register_operand" "w")
1386 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1387 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1388 (match_operand:VQW 4 "register_operand" "w")
1389 (match_dup 3))))))]
1390 "TARGET_SIMD"
1391 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1392 [(set_attr "type" "neon_mla_<Vetype>_long")]
1393 )
1394
1395 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1396 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1397 (minus:<VWIDE>
1398 (match_operand:<VWIDE> 1 "register_operand" "0")
1399 (mult:<VWIDE>
1400 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1401 (match_operand:VQW 2 "register_operand" "w")
1402 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1403 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1404 (match_operand:VQW 4 "register_operand" "w")
1405 (match_dup 3))))))]
1406 "TARGET_SIMD"
1407 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1408 [(set_attr "type" "neon_mla_<Vetype>_long")]
1409 )
1410
1411 (define_insn "*aarch64_<su>mlal<mode>"
1412 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1413 (plus:<VWIDE>
1414 (mult:<VWIDE>
1415 (ANY_EXTEND:<VWIDE>
1416 (match_operand:VD_BHSI 1 "register_operand" "w"))
1417 (ANY_EXTEND:<VWIDE>
1418 (match_operand:VD_BHSI 2 "register_operand" "w")))
1419 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1420 "TARGET_SIMD"
1421 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1422 [(set_attr "type" "neon_mla_<Vetype>_long")]
1423 )
1424
1425 (define_insn "*aarch64_<su>mlsl<mode>"
1426 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1427 (minus:<VWIDE>
1428 (match_operand:<VWIDE> 1 "register_operand" "0")
1429 (mult:<VWIDE>
1430 (ANY_EXTEND:<VWIDE>
1431 (match_operand:VD_BHSI 2 "register_operand" "w"))
1432 (ANY_EXTEND:<VWIDE>
1433 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1434 "TARGET_SIMD"
1435 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1436 [(set_attr "type" "neon_mla_<Vetype>_long")]
1437 )
1438
1439 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1440 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1441 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1442 (match_operand:VQW 1 "register_operand" "w")
1443 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1444 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1445 (match_operand:VQW 2 "register_operand" "w")
1446 (match_dup 3)))))]
1447 "TARGET_SIMD"
1448 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1449 [(set_attr "type" "neon_mul_<Vetype>_long")]
1450 )
1451
1452 (define_expand "vec_widen_<su>mult_lo_<mode>"
1453 [(match_operand:<VWIDE> 0 "register_operand" "")
1454 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1455 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1456 "TARGET_SIMD"
1457 {
1458 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1459 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1460 operands[1],
1461 operands[2], p));
1462 DONE;
1463 }
1464 )
1465
1466 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1467 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1468 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1469 (match_operand:VQW 1 "register_operand" "w")
1470 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1471 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1472 (match_operand:VQW 2 "register_operand" "w")
1473 (match_dup 3)))))]
1474 "TARGET_SIMD"
1475 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1476 [(set_attr "type" "neon_mul_<Vetype>_long")]
1477 )
1478
1479 (define_expand "vec_widen_<su>mult_hi_<mode>"
1480 [(match_operand:<VWIDE> 0 "register_operand" "")
1481 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1482 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1483 "TARGET_SIMD"
1484 {
1485 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1486 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1487 operands[1],
1488 operands[2], p));
1489 DONE;
1490
1491 }
1492 )
1493
1494 ;; FP vector operations.
1495 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1496 ;; double-precision (64-bit) floating-point data types and arithmetic as
1497 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1498 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1499 ;;
1500 ;; Floating-point operations can raise an exception. Vectorizing such
1501 ;; operations are safe because of reasons explained below.
1502 ;;
1503 ;; ARMv8 permits an extension to enable trapped floating-point
1504 ;; exception handling, however this is an optional feature. In the
1505 ;; event of a floating-point exception being raised by vectorised
1506 ;; code then:
1507 ;; 1. If trapped floating-point exceptions are available, then a trap
1508 ;; will be taken when any lane raises an enabled exception. A trap
1509 ;; handler may determine which lane raised the exception.
1510 ;; 2. Alternatively a sticky exception flag is set in the
1511 ;; floating-point status register (FPSR). Software may explicitly
1512 ;; test the exception flags, in which case the tests will either
1513 ;; prevent vectorisation, allowing precise identification of the
1514 ;; failing operation, or if tested outside of vectorisable regions
1515 ;; then the specific operation and lane are not of interest.
1516
1517 ;; FP arithmetic operations.
1518
1519 (define_insn "add<mode>3"
1520 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1521 (plus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1522 (match_operand:VHSDF 2 "register_operand" "w")))]
1523 "TARGET_SIMD"
1524 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1525 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1526 )
1527
1528 (define_insn "sub<mode>3"
1529 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1530 (minus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1531 (match_operand:VHSDF 2 "register_operand" "w")))]
1532 "TARGET_SIMD"
1533 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1534 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1535 )
1536
1537 (define_insn "mul<mode>3"
1538 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1539 (mult:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1540 (match_operand:VHSDF 2 "register_operand" "w")))]
1541 "TARGET_SIMD"
1542 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1543 [(set_attr "type" "neon_fp_mul_<stype><q>")]
1544 )
1545
1546 (define_expand "div<mode>3"
1547 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1548 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1549 (match_operand:VHSDF 2 "register_operand" "w")))]
1550 "TARGET_SIMD"
1551 {
1552 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
1553 DONE;
1554
1555 operands[1] = force_reg (<MODE>mode, operands[1]);
1556 })
1557
1558 (define_insn "*div<mode>3"
1559 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1560 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1561 (match_operand:VHSDF 2 "register_operand" "w")))]
1562 "TARGET_SIMD"
1563 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1564 [(set_attr "type" "neon_fp_div_<stype><q>")]
1565 )
1566
1567 (define_insn "neg<mode>2"
1568 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1569 (neg:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1570 "TARGET_SIMD"
1571 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1572 [(set_attr "type" "neon_fp_neg_<stype><q>")]
1573 )
1574
1575 (define_insn "abs<mode>2"
1576 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1577 (abs:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1578 "TARGET_SIMD"
1579 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1580 [(set_attr "type" "neon_fp_abs_<stype><q>")]
1581 )
1582
1583 (define_insn "fma<mode>4"
1584 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1585 (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1586 (match_operand:VHSDF 2 "register_operand" "w")
1587 (match_operand:VHSDF 3 "register_operand" "0")))]
1588 "TARGET_SIMD"
1589 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1590 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1591 )
1592
1593 (define_insn "*aarch64_fma4_elt<mode>"
1594 [(set (match_operand:VDQF 0 "register_operand" "=w")
1595 (fma:VDQF
1596 (vec_duplicate:VDQF
1597 (vec_select:<VEL>
1598 (match_operand:VDQF 1 "register_operand" "<h_con>")
1599 (parallel [(match_operand:SI 2 "immediate_operand")])))
1600 (match_operand:VDQF 3 "register_operand" "w")
1601 (match_operand:VDQF 4 "register_operand" "0")))]
1602 "TARGET_SIMD"
1603 {
1604 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1605 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1606 }
1607 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1608 )
1609
1610 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1611 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1612 (fma:VDQSF
1613 (vec_duplicate:VDQSF
1614 (vec_select:<VEL>
1615 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1616 (parallel [(match_operand:SI 2 "immediate_operand")])))
1617 (match_operand:VDQSF 3 "register_operand" "w")
1618 (match_operand:VDQSF 4 "register_operand" "0")))]
1619 "TARGET_SIMD"
1620 {
1621 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1622 INTVAL (operands[2])));
1623 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1624 }
1625 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1626 )
1627
1628 (define_insn "*aarch64_fma4_elt_from_dup<mode>"
1629 [(set (match_operand:VMUL 0 "register_operand" "=w")
1630 (fma:VMUL
1631 (vec_duplicate:VMUL
1632 (match_operand:<VEL> 1 "register_operand" "w"))
1633 (match_operand:VMUL 2 "register_operand" "w")
1634 (match_operand:VMUL 3 "register_operand" "0")))]
1635 "TARGET_SIMD"
1636 "fmla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1637 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1638 )
1639
1640 (define_insn "*aarch64_fma4_elt_to_64v2df"
1641 [(set (match_operand:DF 0 "register_operand" "=w")
1642 (fma:DF
1643 (vec_select:DF
1644 (match_operand:V2DF 1 "register_operand" "w")
1645 (parallel [(match_operand:SI 2 "immediate_operand")]))
1646 (match_operand:DF 3 "register_operand" "w")
1647 (match_operand:DF 4 "register_operand" "0")))]
1648 "TARGET_SIMD"
1649 {
1650 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1651 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1652 }
1653 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1654 )
1655
1656 (define_insn "fnma<mode>4"
1657 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1658 (fma:VHSDF
1659 (match_operand:VHSDF 1 "register_operand" "w")
1660 (neg:VHSDF
1661 (match_operand:VHSDF 2 "register_operand" "w"))
1662 (match_operand:VHSDF 3 "register_operand" "0")))]
1663 "TARGET_SIMD"
1664 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1665 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1666 )
1667
1668 (define_insn "*aarch64_fnma4_elt<mode>"
1669 [(set (match_operand:VDQF 0 "register_operand" "=w")
1670 (fma:VDQF
1671 (neg:VDQF
1672 (match_operand:VDQF 3 "register_operand" "w"))
1673 (vec_duplicate:VDQF
1674 (vec_select:<VEL>
1675 (match_operand:VDQF 1 "register_operand" "<h_con>")
1676 (parallel [(match_operand:SI 2 "immediate_operand")])))
1677 (match_operand:VDQF 4 "register_operand" "0")))]
1678 "TARGET_SIMD"
1679 {
1680 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1681 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1682 }
1683 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1684 )
1685
1686 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1687 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1688 (fma:VDQSF
1689 (neg:VDQSF
1690 (match_operand:VDQSF 3 "register_operand" "w"))
1691 (vec_duplicate:VDQSF
1692 (vec_select:<VEL>
1693 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1694 (parallel [(match_operand:SI 2 "immediate_operand")])))
1695 (match_operand:VDQSF 4 "register_operand" "0")))]
1696 "TARGET_SIMD"
1697 {
1698 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1699 INTVAL (operands[2])));
1700 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1701 }
1702 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1703 )
1704
1705 (define_insn "*aarch64_fnma4_elt_from_dup<mode>"
1706 [(set (match_operand:VMUL 0 "register_operand" "=w")
1707 (fma:VMUL
1708 (neg:VMUL
1709 (match_operand:VMUL 2 "register_operand" "w"))
1710 (vec_duplicate:VMUL
1711 (match_operand:<VEL> 1 "register_operand" "w"))
1712 (match_operand:VMUL 3 "register_operand" "0")))]
1713 "TARGET_SIMD"
1714 "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1715 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1716 )
1717
1718 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1719 [(set (match_operand:DF 0 "register_operand" "=w")
1720 (fma:DF
1721 (vec_select:DF
1722 (match_operand:V2DF 1 "register_operand" "w")
1723 (parallel [(match_operand:SI 2 "immediate_operand")]))
1724 (neg:DF
1725 (match_operand:DF 3 "register_operand" "w"))
1726 (match_operand:DF 4 "register_operand" "0")))]
1727 "TARGET_SIMD"
1728 {
1729 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1730 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1731 }
1732 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1733 )
1734
1735 ;; Vector versions of the floating-point frint patterns.
1736 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1737 (define_insn "<frint_pattern><mode>2"
1738 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1739 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
1740 FRINT))]
1741 "TARGET_SIMD"
1742 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1743 [(set_attr "type" "neon_fp_round_<stype><q>")]
1744 )
1745
1746 ;; Vector versions of the fcvt standard patterns.
1747 ;; Expands to lbtrunc, lround, lceil, lfloor
1748 (define_insn "l<fcvt_pattern><su_optab><VHSDF:mode><fcvt_target>2"
1749 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1750 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1751 [(match_operand:VHSDF 1 "register_operand" "w")]
1752 FCVT)))]
1753 "TARGET_SIMD"
1754 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1755 [(set_attr "type" "neon_fp_to_int_<stype><q>")]
1756 )
1757
1758 ;; HF Scalar variants of related SIMD instructions.
1759 (define_insn "l<fcvt_pattern><su_optab>hfhi2"
1760 [(set (match_operand:HI 0 "register_operand" "=w")
1761 (FIXUORS:HI (unspec:HF [(match_operand:HF 1 "register_operand" "w")]
1762 FCVT)))]
1763 "TARGET_SIMD_F16INST"
1764 "fcvt<frint_suffix><su>\t%h0, %h1"
1765 [(set_attr "type" "neon_fp_to_int_s")]
1766 )
1767
1768 (define_insn "<optab>_trunchfhi2"
1769 [(set (match_operand:HI 0 "register_operand" "=w")
1770 (FIXUORS:HI (match_operand:HF 1 "register_operand" "w")))]
1771 "TARGET_SIMD_F16INST"
1772 "fcvtz<su>\t%h0, %h1"
1773 [(set_attr "type" "neon_fp_to_int_s")]
1774 )
1775
1776 (define_insn "<optab>hihf2"
1777 [(set (match_operand:HF 0 "register_operand" "=w")
1778 (FLOATUORS:HF (match_operand:HI 1 "register_operand" "w")))]
1779 "TARGET_SIMD_F16INST"
1780 "<su_optab>cvtf\t%h0, %h1"
1781 [(set_attr "type" "neon_int_to_fp_s")]
1782 )
1783
1784 (define_insn "*aarch64_fcvt<su_optab><VDQF:mode><fcvt_target>2_mult"
1785 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1786 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1787 [(mult:VDQF
1788 (match_operand:VDQF 1 "register_operand" "w")
1789 (match_operand:VDQF 2 "aarch64_fp_vec_pow2" ""))]
1790 UNSPEC_FRINTZ)))]
1791 "TARGET_SIMD
1792 && IN_RANGE (aarch64_vec_fpconst_pow_of_2 (operands[2]), 1,
1793 GET_MODE_BITSIZE (GET_MODE_INNER (<VDQF:MODE>mode)))"
1794 {
1795 int fbits = aarch64_vec_fpconst_pow_of_2 (operands[2]);
1796 char buf[64];
1797 snprintf (buf, 64, "fcvtz<su>\\t%%0.<Vtype>, %%1.<Vtype>, #%d", fbits);
1798 output_asm_insn (buf, operands);
1799 return "";
1800 }
1801 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1802 )
1803
1804 (define_expand "<optab><VHSDF:mode><fcvt_target>2"
1805 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1806 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1807 [(match_operand:VHSDF 1 "register_operand")]
1808 UNSPEC_FRINTZ)))]
1809 "TARGET_SIMD"
1810 {})
1811
1812 (define_expand "<fix_trunc_optab><VHSDF:mode><fcvt_target>2"
1813 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1814 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1815 [(match_operand:VHSDF 1 "register_operand")]
1816 UNSPEC_FRINTZ)))]
1817 "TARGET_SIMD"
1818 {})
1819
1820 (define_expand "ftrunc<VHSDF:mode>2"
1821 [(set (match_operand:VHSDF 0 "register_operand")
1822 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
1823 UNSPEC_FRINTZ))]
1824 "TARGET_SIMD"
1825 {})
1826
1827 (define_insn "<optab><fcvt_target><VHSDF:mode>2"
1828 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1829 (FLOATUORS:VHSDF
1830 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1831 "TARGET_SIMD"
1832 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1833 [(set_attr "type" "neon_int_to_fp_<stype><q>")]
1834 )
1835
1836 ;; Conversions between vectors of floats and doubles.
1837 ;; Contains a mix of patterns to match standard pattern names
1838 ;; and those for intrinsics.
1839
1840 ;; Float widening operations.
1841
1842 (define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
1843 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1844 (float_extend:<VWIDE> (vec_select:<VHALF>
1845 (match_operand:VQ_HSF 1 "register_operand" "w")
1846 (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
1847 )))]
1848 "TARGET_SIMD"
1849 "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
1850 [(set_attr "type" "neon_fp_cvt_widen_s")]
1851 )
1852
1853 ;; Convert between fixed-point and floating-point (vector modes)
1854
1855 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><VHSDF:mode>3"
1856 [(set (match_operand:<VHSDF:FCVT_TARGET> 0 "register_operand" "=w")
1857 (unspec:<VHSDF:FCVT_TARGET>
1858 [(match_operand:VHSDF 1 "register_operand" "w")
1859 (match_operand:SI 2 "immediate_operand" "i")]
1860 FCVT_F2FIXED))]
1861 "TARGET_SIMD"
1862 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
1863 [(set_attr "type" "neon_fp_to_int_<VHSDF:stype><q>")]
1864 )
1865
1866 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><VDQ_HSDI:mode>3"
1867 [(set (match_operand:<VDQ_HSDI:FCVT_TARGET> 0 "register_operand" "=w")
1868 (unspec:<VDQ_HSDI:FCVT_TARGET>
1869 [(match_operand:VDQ_HSDI 1 "register_operand" "w")
1870 (match_operand:SI 2 "immediate_operand" "i")]
1871 FCVT_FIXED2F))]
1872 "TARGET_SIMD"
1873 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
1874 [(set_attr "type" "neon_int_to_fp_<VDQ_HSDI:stype><q>")]
1875 )
1876
1877 ;; ??? Note that the vectorizer usage of the vec_unpacks_[lo/hi] patterns
1878 ;; is inconsistent with vector ordering elsewhere in the compiler, in that
1879 ;; the meaning of HI and LO changes depending on the target endianness.
1880 ;; While elsewhere we map the higher numbered elements of a vector to
1881 ;; the lower architectural lanes of the vector, for these patterns we want
1882 ;; to always treat "hi" as referring to the higher architectural lanes.
1883 ;; Consequently, while the patterns below look inconsistent with our
1884 ;; other big-endian patterns their behavior is as required.
1885
1886 (define_expand "vec_unpacks_lo_<mode>"
1887 [(match_operand:<VWIDE> 0 "register_operand" "")
1888 (match_operand:VQ_HSF 1 "register_operand" "")]
1889 "TARGET_SIMD"
1890 {
1891 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1892 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1893 operands[1], p));
1894 DONE;
1895 }
1896 )
1897
1898 (define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
1899 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1900 (float_extend:<VWIDE> (vec_select:<VHALF>
1901 (match_operand:VQ_HSF 1 "register_operand" "w")
1902 (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
1903 )))]
1904 "TARGET_SIMD"
1905 "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
1906 [(set_attr "type" "neon_fp_cvt_widen_s")]
1907 )
1908
1909 (define_expand "vec_unpacks_hi_<mode>"
1910 [(match_operand:<VWIDE> 0 "register_operand" "")
1911 (match_operand:VQ_HSF 1 "register_operand" "")]
1912 "TARGET_SIMD"
1913 {
1914 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1915 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1916 operands[1], p));
1917 DONE;
1918 }
1919 )
1920 (define_insn "aarch64_float_extend_lo_<Vwide>"
1921 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1922 (float_extend:<VWIDE>
1923 (match_operand:VDF 1 "register_operand" "w")))]
1924 "TARGET_SIMD"
1925 "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
1926 [(set_attr "type" "neon_fp_cvt_widen_s")]
1927 )
1928
1929 ;; Float narrowing operations.
1930
1931 (define_insn "aarch64_float_truncate_lo_<mode>"
1932 [(set (match_operand:VDF 0 "register_operand" "=w")
1933 (float_truncate:VDF
1934 (match_operand:<VWIDE> 1 "register_operand" "w")))]
1935 "TARGET_SIMD"
1936 "fcvtn\\t%0.<Vtype>, %1<Vmwtype>"
1937 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1938 )
1939
1940 (define_insn "aarch64_float_truncate_hi_<Vdbl>_le"
1941 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1942 (vec_concat:<VDBL>
1943 (match_operand:VDF 1 "register_operand" "0")
1944 (float_truncate:VDF
1945 (match_operand:<VWIDE> 2 "register_operand" "w"))))]
1946 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1947 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1948 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1949 )
1950
1951 (define_insn "aarch64_float_truncate_hi_<Vdbl>_be"
1952 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1953 (vec_concat:<VDBL>
1954 (float_truncate:VDF
1955 (match_operand:<VWIDE> 2 "register_operand" "w"))
1956 (match_operand:VDF 1 "register_operand" "0")))]
1957 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1958 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1959 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1960 )
1961
1962 (define_expand "aarch64_float_truncate_hi_<Vdbl>"
1963 [(match_operand:<VDBL> 0 "register_operand" "=w")
1964 (match_operand:VDF 1 "register_operand" "0")
1965 (match_operand:<VWIDE> 2 "register_operand" "w")]
1966 "TARGET_SIMD"
1967 {
1968 rtx (*gen) (rtx, rtx, rtx) = BYTES_BIG_ENDIAN
1969 ? gen_aarch64_float_truncate_hi_<Vdbl>_be
1970 : gen_aarch64_float_truncate_hi_<Vdbl>_le;
1971 emit_insn (gen (operands[0], operands[1], operands[2]));
1972 DONE;
1973 }
1974 )
1975
1976 (define_expand "vec_pack_trunc_v2df"
1977 [(set (match_operand:V4SF 0 "register_operand")
1978 (vec_concat:V4SF
1979 (float_truncate:V2SF
1980 (match_operand:V2DF 1 "register_operand"))
1981 (float_truncate:V2SF
1982 (match_operand:V2DF 2 "register_operand"))
1983 ))]
1984 "TARGET_SIMD"
1985 {
1986 rtx tmp = gen_reg_rtx (V2SFmode);
1987 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1988 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1989
1990 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1991 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1992 tmp, operands[hi]));
1993 DONE;
1994 }
1995 )
1996
1997 (define_expand "vec_pack_trunc_df"
1998 [(set (match_operand:V2SF 0 "register_operand")
1999 (vec_concat:V2SF
2000 (float_truncate:SF
2001 (match_operand:DF 1 "register_operand"))
2002 (float_truncate:SF
2003 (match_operand:DF 2 "register_operand"))
2004 ))]
2005 "TARGET_SIMD"
2006 {
2007 rtx tmp = gen_reg_rtx (V2SFmode);
2008 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2009 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2010
2011 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
2012 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
2013 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
2014 DONE;
2015 }
2016 )
2017
2018 ;; FP Max/Min
2019 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
2020 ;; expression like:
2021 ;; a = (b < c) ? b : c;
2022 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
2023 ;; either explicitly or indirectly via -ffast-math.
2024 ;;
2025 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
2026 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
2027 ;; operand will be returned when both operands are zero (i.e. they may not
2028 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
2029 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
2030 ;; NaNs.
2031
2032 (define_insn "<su><maxmin><mode>3"
2033 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2034 (FMAXMIN:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
2035 (match_operand:VHSDF 2 "register_operand" "w")))]
2036 "TARGET_SIMD"
2037 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2038 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2039 )
2040
2041 ;; Vector forms for fmax, fmin, fmaxnm, fminnm.
2042 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
2043 ;; which implement the IEEE fmax ()/fmin () functions.
2044 (define_insn "<maxmin_uns><mode>3"
2045 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2046 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2047 (match_operand:VHSDF 2 "register_operand" "w")]
2048 FMAXMIN_UNS))]
2049 "TARGET_SIMD"
2050 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2051 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2052 )
2053
2054 ;; 'across lanes' add.
2055
2056 (define_expand "reduc_plus_scal_<mode>"
2057 [(match_operand:<VEL> 0 "register_operand" "=w")
2058 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
2059 UNSPEC_ADDV)]
2060 "TARGET_SIMD"
2061 {
2062 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2063 rtx scratch = gen_reg_rtx (<MODE>mode);
2064 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
2065 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2066 DONE;
2067 }
2068 )
2069
2070 (define_insn "aarch64_faddp<mode>"
2071 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2072 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2073 (match_operand:VHSDF 2 "register_operand" "w")]
2074 UNSPEC_FADDV))]
2075 "TARGET_SIMD"
2076 "faddp\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2077 [(set_attr "type" "neon_fp_reduc_add_<stype><q>")]
2078 )
2079
2080 (define_insn "aarch64_reduc_plus_internal<mode>"
2081 [(set (match_operand:VDQV 0 "register_operand" "=w")
2082 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
2083 UNSPEC_ADDV))]
2084 "TARGET_SIMD"
2085 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
2086 [(set_attr "type" "neon_reduc_add<q>")]
2087 )
2088
2089 (define_insn "aarch64_reduc_plus_internalv2si"
2090 [(set (match_operand:V2SI 0 "register_operand" "=w")
2091 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2092 UNSPEC_ADDV))]
2093 "TARGET_SIMD"
2094 "addp\\t%0.2s, %1.2s, %1.2s"
2095 [(set_attr "type" "neon_reduc_add")]
2096 )
2097
2098 (define_insn "reduc_plus_scal_<mode>"
2099 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2100 (unspec:<VEL> [(match_operand:V2F 1 "register_operand" "w")]
2101 UNSPEC_FADDV))]
2102 "TARGET_SIMD"
2103 "faddp\\t%<Vetype>0, %1.<Vtype>"
2104 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
2105 )
2106
2107 (define_expand "reduc_plus_scal_v4sf"
2108 [(set (match_operand:SF 0 "register_operand")
2109 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
2110 UNSPEC_FADDV))]
2111 "TARGET_SIMD"
2112 {
2113 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
2114 rtx scratch = gen_reg_rtx (V4SFmode);
2115 emit_insn (gen_aarch64_faddpv4sf (scratch, operands[1], operands[1]));
2116 emit_insn (gen_aarch64_faddpv4sf (scratch, scratch, scratch));
2117 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
2118 DONE;
2119 })
2120
2121 (define_insn "clrsb<mode>2"
2122 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2123 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2124 "TARGET_SIMD"
2125 "cls\\t%0.<Vtype>, %1.<Vtype>"
2126 [(set_attr "type" "neon_cls<q>")]
2127 )
2128
2129 (define_insn "clz<mode>2"
2130 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2131 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2132 "TARGET_SIMD"
2133 "clz\\t%0.<Vtype>, %1.<Vtype>"
2134 [(set_attr "type" "neon_cls<q>")]
2135 )
2136
2137 (define_insn "popcount<mode>2"
2138 [(set (match_operand:VB 0 "register_operand" "=w")
2139 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
2140 "TARGET_SIMD"
2141 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
2142 [(set_attr "type" "neon_cnt<q>")]
2143 )
2144
2145 ;; 'across lanes' max and min ops.
2146
2147 ;; Template for outputting a scalar, so we can create __builtins which can be
2148 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
2149 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2150 [(match_operand:<VEL> 0 "register_operand")
2151 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
2152 FMAXMINV)]
2153 "TARGET_SIMD"
2154 {
2155 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2156 rtx scratch = gen_reg_rtx (<MODE>mode);
2157 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2158 operands[1]));
2159 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2160 DONE;
2161 }
2162 )
2163
2164 ;; Likewise for integer cases, signed and unsigned.
2165 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2166 [(match_operand:<VEL> 0 "register_operand")
2167 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
2168 MAXMINV)]
2169 "TARGET_SIMD"
2170 {
2171 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2172 rtx scratch = gen_reg_rtx (<MODE>mode);
2173 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2174 operands[1]));
2175 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2176 DONE;
2177 }
2178 )
2179
2180 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2181 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
2182 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
2183 MAXMINV))]
2184 "TARGET_SIMD"
2185 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
2186 [(set_attr "type" "neon_reduc_minmax<q>")]
2187 )
2188
2189 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
2190 [(set (match_operand:V2SI 0 "register_operand" "=w")
2191 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2192 MAXMINV))]
2193 "TARGET_SIMD"
2194 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
2195 [(set_attr "type" "neon_reduc_minmax")]
2196 )
2197
2198 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2199 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2200 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
2201 FMAXMINV))]
2202 "TARGET_SIMD"
2203 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
2204 [(set_attr "type" "neon_fp_reduc_minmax_<stype><q>")]
2205 )
2206
2207 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
2208 ;; allocation.
2209 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
2210 ;; to select.
2211 ;;
2212 ;; Thus our BSL is of the form:
2213 ;; op0 = bsl (mask, op2, op3)
2214 ;; We can use any of:
2215 ;;
2216 ;; if (op0 = mask)
2217 ;; bsl mask, op1, op2
2218 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
2219 ;; bit op0, op2, mask
2220 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
2221 ;; bif op0, op1, mask
2222 ;;
2223 ;; This pattern is expanded to by the aarch64_simd_bsl<mode> expander.
2224 ;; Some forms of straight-line code may generate the equivalent form
2225 ;; in *aarch64_simd_bsl<mode>_alt.
2226
2227 (define_insn "aarch64_simd_bsl<mode>_internal"
2228 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2229 (xor:VSDQ_I_DI
2230 (and:VSDQ_I_DI
2231 (xor:VSDQ_I_DI
2232 (match_operand:<V_cmp_result> 3 "register_operand" "w,0,w")
2233 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2234 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2235 (match_dup:<V_cmp_result> 3)
2236 ))]
2237 "TARGET_SIMD"
2238 "@
2239 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2240 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2241 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2242 [(set_attr "type" "neon_bsl<q>")]
2243 )
2244
2245 ;; We need this form in addition to the above pattern to match the case
2246 ;; when combine tries merging three insns such that the second operand of
2247 ;; the outer XOR matches the second operand of the inner XOR rather than
2248 ;; the first. The two are equivalent but since recog doesn't try all
2249 ;; permutations of commutative operations, we have to have a separate pattern.
2250
2251 (define_insn "*aarch64_simd_bsl<mode>_alt"
2252 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2253 (xor:VSDQ_I_DI
2254 (and:VSDQ_I_DI
2255 (xor:VSDQ_I_DI
2256 (match_operand:VSDQ_I_DI 3 "register_operand" "w,w,0")
2257 (match_operand:VSDQ_I_DI 2 "register_operand" "w,0,w"))
2258 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2259 (match_dup:VSDQ_I_DI 2)))]
2260 "TARGET_SIMD"
2261 "@
2262 bsl\\t%0.<Vbtype>, %3.<Vbtype>, %2.<Vbtype>
2263 bit\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>
2264 bif\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
2265 [(set_attr "type" "neon_bsl<q>")]
2266 )
2267
2268 (define_expand "aarch64_simd_bsl<mode>"
2269 [(match_operand:VALLDIF 0 "register_operand")
2270 (match_operand:<V_cmp_result> 1 "register_operand")
2271 (match_operand:VALLDIF 2 "register_operand")
2272 (match_operand:VALLDIF 3 "register_operand")]
2273 "TARGET_SIMD"
2274 {
2275 /* We can't alias operands together if they have different modes. */
2276 rtx tmp = operands[0];
2277 if (FLOAT_MODE_P (<MODE>mode))
2278 {
2279 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2280 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2281 tmp = gen_reg_rtx (<V_cmp_result>mode);
2282 }
2283 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
2284 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2285 operands[1],
2286 operands[2],
2287 operands[3]));
2288 if (tmp != operands[0])
2289 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2290
2291 DONE;
2292 })
2293
2294 (define_expand "vcond_mask_<mode><v_cmp_result>"
2295 [(match_operand:VALLDI 0 "register_operand")
2296 (match_operand:VALLDI 1 "nonmemory_operand")
2297 (match_operand:VALLDI 2 "nonmemory_operand")
2298 (match_operand:<V_cmp_result> 3 "register_operand")]
2299 "TARGET_SIMD"
2300 {
2301 /* If we have (a = (P) ? -1 : 0);
2302 Then we can simply move the generated mask (result must be int). */
2303 if (operands[1] == CONSTM1_RTX (<MODE>mode)
2304 && operands[2] == CONST0_RTX (<MODE>mode))
2305 emit_move_insn (operands[0], operands[3]);
2306 /* Similarly, (a = (P) ? 0 : -1) is just inverting the generated mask. */
2307 else if (operands[1] == CONST0_RTX (<MODE>mode)
2308 && operands[2] == CONSTM1_RTX (<MODE>mode))
2309 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[3]));
2310 else
2311 {
2312 if (!REG_P (operands[1]))
2313 operands[1] = force_reg (<MODE>mode, operands[1]);
2314 if (!REG_P (operands[2]))
2315 operands[2] = force_reg (<MODE>mode, operands[2]);
2316 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], operands[3],
2317 operands[1], operands[2]));
2318 }
2319
2320 DONE;
2321 })
2322
2323 ;; Patterns comparing two vectors to produce a mask.
2324
2325 (define_expand "vec_cmp<mode><mode>"
2326 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2327 (match_operator 1 "comparison_operator"
2328 [(match_operand:VSDQ_I_DI 2 "register_operand")
2329 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2330 "TARGET_SIMD"
2331 {
2332 rtx mask = operands[0];
2333 enum rtx_code code = GET_CODE (operands[1]);
2334
2335 switch (code)
2336 {
2337 case NE:
2338 case LE:
2339 case LT:
2340 case GE:
2341 case GT:
2342 case EQ:
2343 if (operands[3] == CONST0_RTX (<MODE>mode))
2344 break;
2345
2346 /* Fall through. */
2347 default:
2348 if (!REG_P (operands[3]))
2349 operands[3] = force_reg (<MODE>mode, operands[3]);
2350
2351 break;
2352 }
2353
2354 switch (code)
2355 {
2356 case LT:
2357 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[2], operands[3]));
2358 break;
2359
2360 case GE:
2361 emit_insn (gen_aarch64_cmge<mode> (mask, operands[2], operands[3]));
2362 break;
2363
2364 case LE:
2365 emit_insn (gen_aarch64_cmle<mode> (mask, operands[2], operands[3]));
2366 break;
2367
2368 case GT:
2369 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[2], operands[3]));
2370 break;
2371
2372 case LTU:
2373 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[3], operands[2]));
2374 break;
2375
2376 case GEU:
2377 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[2], operands[3]));
2378 break;
2379
2380 case LEU:
2381 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[3], operands[2]));
2382 break;
2383
2384 case GTU:
2385 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[2], operands[3]));
2386 break;
2387
2388 case NE:
2389 /* Handle NE as !EQ. */
2390 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2391 emit_insn (gen_one_cmpl<v_cmp_result>2 (mask, mask));
2392 break;
2393
2394 case EQ:
2395 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2396 break;
2397
2398 default:
2399 gcc_unreachable ();
2400 }
2401
2402 DONE;
2403 })
2404
2405 (define_expand "vec_cmp<mode><v_cmp_result>"
2406 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2407 (match_operator 1 "comparison_operator"
2408 [(match_operand:VDQF 2 "register_operand")
2409 (match_operand:VDQF 3 "nonmemory_operand")]))]
2410 "TARGET_SIMD"
2411 {
2412 int use_zero_form = 0;
2413 enum rtx_code code = GET_CODE (operands[1]);
2414 rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
2415
2416 rtx (*comparison) (rtx, rtx, rtx);
2417
2418 switch (code)
2419 {
2420 case LE:
2421 case LT:
2422 case GE:
2423 case GT:
2424 case EQ:
2425 if (operands[3] == CONST0_RTX (<MODE>mode))
2426 {
2427 use_zero_form = 1;
2428 break;
2429 }
2430 /* Fall through. */
2431 default:
2432 if (!REG_P (operands[3]))
2433 operands[3] = force_reg (<MODE>mode, operands[3]);
2434
2435 break;
2436 }
2437
2438 switch (code)
2439 {
2440 case LT:
2441 if (use_zero_form)
2442 {
2443 comparison = gen_aarch64_cmlt<mode>;
2444 break;
2445 }
2446 /* Else, fall through. */
2447 case UNGE:
2448 std::swap (operands[2], operands[3]);
2449 /* Fall through. */
2450 case UNLE:
2451 case GT:
2452 comparison = gen_aarch64_cmgt<mode>;
2453 break;
2454 case LE:
2455 if (use_zero_form)
2456 {
2457 comparison = gen_aarch64_cmle<mode>;
2458 break;
2459 }
2460 /* Else, fall through. */
2461 case UNGT:
2462 std::swap (operands[2], operands[3]);
2463 /* Fall through. */
2464 case UNLT:
2465 case GE:
2466 comparison = gen_aarch64_cmge<mode>;
2467 break;
2468 case NE:
2469 case EQ:
2470 comparison = gen_aarch64_cmeq<mode>;
2471 break;
2472 case UNEQ:
2473 case ORDERED:
2474 case UNORDERED:
2475 break;
2476 default:
2477 gcc_unreachable ();
2478 }
2479
2480 switch (code)
2481 {
2482 case UNGE:
2483 case UNGT:
2484 case UNLE:
2485 case UNLT:
2486 case NE:
2487 /* FCM returns false for lanes which are unordered, so if we use
2488 the inverse of the comparison we actually want to emit, then
2489 invert the result, we will end up with the correct result.
2490 Note that a NE NaN and NaN NE b are true for all a, b.
2491
2492 Our transformations are:
2493 a UNGE b -> !(b GT a)
2494 a UNGT b -> !(b GE a)
2495 a UNLE b -> !(a GT b)
2496 a UNLT b -> !(a GE b)
2497 a NE b -> !(a EQ b) */
2498 emit_insn (comparison (operands[0], operands[2], operands[3]));
2499 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[0]));
2500 break;
2501
2502 case LT:
2503 case LE:
2504 case GT:
2505 case GE:
2506 case EQ:
2507 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2508 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2509 a GE b -> a GE b
2510 a GT b -> a GT b
2511 a LE b -> b GE a
2512 a LT b -> b GT a
2513 a EQ b -> a EQ b */
2514 emit_insn (comparison (operands[0], operands[2], operands[3]));
2515 break;
2516
2517 case UNEQ:
2518 /* We first check (a > b || b > a) which is !UNEQ, inverting
2519 this result will then give us (a == b || a UNORDERED b). */
2520 emit_insn (gen_aarch64_cmgt<mode> (operands[0],
2521 operands[2], operands[3]));
2522 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[3], operands[2]));
2523 emit_insn (gen_ior<v_cmp_result>3 (operands[0], operands[0], tmp));
2524 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[0]));
2525 break;
2526
2527 case UNORDERED:
2528 /* Operands are ORDERED iff (a > b || b >= a), so we can compute
2529 UNORDERED as !ORDERED. */
2530 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2531 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2532 operands[3], operands[2]));
2533 emit_insn (gen_ior<v_cmp_result>3 (operands[0], operands[0], tmp));
2534 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[0]));
2535 break;
2536
2537 case ORDERED:
2538 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2539 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2540 operands[3], operands[2]));
2541 emit_insn (gen_ior<v_cmp_result>3 (operands[0], operands[0], tmp));
2542 break;
2543
2544 default:
2545 gcc_unreachable ();
2546 }
2547
2548 DONE;
2549 })
2550
2551 (define_expand "vec_cmpu<mode><mode>"
2552 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2553 (match_operator 1 "comparison_operator"
2554 [(match_operand:VSDQ_I_DI 2 "register_operand")
2555 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2556 "TARGET_SIMD"
2557 {
2558 emit_insn (gen_vec_cmp<mode><mode> (operands[0], operands[1],
2559 operands[2], operands[3]));
2560 DONE;
2561 })
2562
2563 (define_expand "vcond<mode><mode>"
2564 [(set (match_operand:VALLDI 0 "register_operand")
2565 (if_then_else:VALLDI
2566 (match_operator 3 "comparison_operator"
2567 [(match_operand:VALLDI 4 "register_operand")
2568 (match_operand:VALLDI 5 "nonmemory_operand")])
2569 (match_operand:VALLDI 1 "nonmemory_operand")
2570 (match_operand:VALLDI 2 "nonmemory_operand")))]
2571 "TARGET_SIMD"
2572 {
2573 rtx mask = gen_reg_rtx (<V_cmp_result>mode);
2574 enum rtx_code code = GET_CODE (operands[3]);
2575
2576 emit_insn (gen_vec_cmp<mode><v_cmp_result> (mask, operands[3],
2577 operands[4], operands[5]));
2578 emit_insn (gen_vcond_mask_<mode><v_cmp_result> (operands[0], operands[1],
2579 operands[2], mask));
2580
2581 DONE;
2582 })
2583
2584 (define_expand "vcond<v_cmp_mixed><mode>"
2585 [(set (match_operand:<V_cmp_mixed> 0 "register_operand")
2586 (if_then_else:<V_cmp_mixed>
2587 (match_operator 3 "comparison_operator"
2588 [(match_operand:VDQF_COND 4 "register_operand")
2589 (match_operand:VDQF_COND 5 "nonmemory_operand")])
2590 (match_operand:<V_cmp_mixed> 1 "nonmemory_operand")
2591 (match_operand:<V_cmp_mixed> 2 "nonmemory_operand")))]
2592 "TARGET_SIMD"
2593 {
2594 rtx mask = gen_reg_rtx (<V_cmp_result>mode);
2595 enum rtx_code code = GET_CODE (operands[3]);
2596
2597 emit_insn (gen_vec_cmp<mode><v_cmp_result> (mask, operands[3],
2598 operands[4], operands[5]));
2599 emit_insn (gen_vcond_mask_<v_cmp_mixed><v_cmp_result> (
2600 operands[0], operands[1],
2601 operands[2], mask));
2602
2603 DONE;
2604 })
2605
2606 (define_expand "vcondu<mode><mode>"
2607 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2608 (if_then_else:VSDQ_I_DI
2609 (match_operator 3 "comparison_operator"
2610 [(match_operand:VSDQ_I_DI 4 "register_operand")
2611 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2612 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2613 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2614 "TARGET_SIMD"
2615 {
2616 rtx mask = gen_reg_rtx (<MODE>mode);
2617 enum rtx_code code = GET_CODE (operands[3]);
2618
2619 emit_insn (gen_vec_cmp<mode><mode> (mask, operands[3],
2620 operands[4], operands[5]));
2621 emit_insn (gen_vcond_mask_<mode><v_cmp_result> (operands[0], operands[1],
2622 operands[2], mask));
2623 DONE;
2624 })
2625
2626 (define_expand "vcondu<mode><v_cmp_mixed>"
2627 [(set (match_operand:VDQF 0 "register_operand")
2628 (if_then_else:VDQF
2629 (match_operator 3 "comparison_operator"
2630 [(match_operand:<V_cmp_mixed> 4 "register_operand")
2631 (match_operand:<V_cmp_mixed> 5 "nonmemory_operand")])
2632 (match_operand:VDQF 1 "nonmemory_operand")
2633 (match_operand:VDQF 2 "nonmemory_operand")))]
2634 "TARGET_SIMD"
2635 {
2636 rtx mask = gen_reg_rtx (<V_cmp_result>mode);
2637 enum rtx_code code = GET_CODE (operands[3]);
2638
2639 emit_insn (gen_vec_cmp<v_cmp_mixed><v_cmp_mixed> (
2640 mask, operands[3],
2641 operands[4], operands[5]));
2642 emit_insn (gen_vcond_mask_<mode><v_cmp_result> (operands[0], operands[1],
2643 operands[2], mask));
2644 DONE;
2645 })
2646
2647 ;; Patterns for AArch64 SIMD Intrinsics.
2648
2649 ;; Lane extraction with sign extension to general purpose register.
2650 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2651 [(set (match_operand:GPI 0 "register_operand" "=r")
2652 (sign_extend:GPI
2653 (vec_select:<VEL>
2654 (match_operand:VDQQH 1 "register_operand" "w")
2655 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2656 "TARGET_SIMD"
2657 {
2658 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2659 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2660 }
2661 [(set_attr "type" "neon_to_gp<q>")]
2662 )
2663
2664 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2665 [(set (match_operand:SI 0 "register_operand" "=r")
2666 (zero_extend:SI
2667 (vec_select:<VEL>
2668 (match_operand:VDQQH 1 "register_operand" "w")
2669 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2670 "TARGET_SIMD"
2671 {
2672 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2673 return "umov\\t%w0, %1.<Vetype>[%2]";
2674 }
2675 [(set_attr "type" "neon_to_gp<q>")]
2676 )
2677
2678 ;; Lane extraction of a value, neither sign nor zero extension
2679 ;; is guaranteed so upper bits should be considered undefined.
2680 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2681 (define_insn "aarch64_get_lane<mode>"
2682 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2683 (vec_select:<VEL>
2684 (match_operand:VALL_F16 1 "register_operand" "w, w, w")
2685 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2686 "TARGET_SIMD"
2687 {
2688 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2689 switch (which_alternative)
2690 {
2691 case 0:
2692 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2693 case 1:
2694 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2695 case 2:
2696 return "st1\\t{%1.<Vetype>}[%2], %0";
2697 default:
2698 gcc_unreachable ();
2699 }
2700 }
2701 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2702 )
2703
2704 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2705 ;; dest vector.
2706
2707 (define_insn "*aarch64_combinez<mode>"
2708 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2709 (vec_concat:<VDBL>
2710 (match_operand:VD_BHSI 1 "general_operand" "w,?r,m")
2711 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")))]
2712 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2713 "@
2714 mov\\t%0.8b, %1.8b
2715 fmov\t%d0, %1
2716 ldr\\t%d0, %1"
2717 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2718 (set_attr "simd" "yes,*,yes")
2719 (set_attr "fp" "*,yes,*")]
2720 )
2721
2722 (define_insn "*aarch64_combinez_be<mode>"
2723 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2724 (vec_concat:<VDBL>
2725 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")
2726 (match_operand:VD_BHSI 1 "general_operand" "w,?r,m")))]
2727 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2728 "@
2729 mov\\t%0.8b, %1.8b
2730 fmov\t%d0, %1
2731 ldr\\t%d0, %1"
2732 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2733 (set_attr "simd" "yes,*,yes")
2734 (set_attr "fp" "*,yes,*")]
2735 )
2736
2737 (define_expand "aarch64_combine<mode>"
2738 [(match_operand:<VDBL> 0 "register_operand")
2739 (match_operand:VDC 1 "register_operand")
2740 (match_operand:VDC 2 "register_operand")]
2741 "TARGET_SIMD"
2742 {
2743 rtx op1, op2;
2744 if (BYTES_BIG_ENDIAN)
2745 {
2746 op1 = operands[2];
2747 op2 = operands[1];
2748 }
2749 else
2750 {
2751 op1 = operands[1];
2752 op2 = operands[2];
2753 }
2754 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2755 DONE;
2756 }
2757 )
2758
2759 (define_insn_and_split "aarch64_combine_internal<mode>"
2760 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2761 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2762 (match_operand:VDC 2 "register_operand" "w")))]
2763 "TARGET_SIMD"
2764 "#"
2765 "&& reload_completed"
2766 [(const_int 0)]
2767 {
2768 if (BYTES_BIG_ENDIAN)
2769 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2770 else
2771 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2772 DONE;
2773 }
2774 [(set_attr "type" "multiple")]
2775 )
2776
2777 (define_expand "aarch64_simd_combine<mode>"
2778 [(match_operand:<VDBL> 0 "register_operand")
2779 (match_operand:VDC 1 "register_operand")
2780 (match_operand:VDC 2 "register_operand")]
2781 "TARGET_SIMD"
2782 {
2783 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2784 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2785 DONE;
2786 }
2787 [(set_attr "type" "multiple")]
2788 )
2789
2790 ;; <su><addsub>l<q>.
2791
2792 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2793 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2794 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2795 (match_operand:VQW 1 "register_operand" "w")
2796 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2797 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2798 (match_operand:VQW 2 "register_operand" "w")
2799 (match_dup 3)))))]
2800 "TARGET_SIMD"
2801 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2802 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2803 )
2804
2805 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2806 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2807 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2808 (match_operand:VQW 1 "register_operand" "w")
2809 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2810 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2811 (match_operand:VQW 2 "register_operand" "w")
2812 (match_dup 3)))))]
2813 "TARGET_SIMD"
2814 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2815 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2816 )
2817
2818
2819 (define_expand "aarch64_saddl2<mode>"
2820 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2821 (match_operand:VQW 1 "register_operand" "w")
2822 (match_operand:VQW 2 "register_operand" "w")]
2823 "TARGET_SIMD"
2824 {
2825 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2826 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2827 operands[2], p));
2828 DONE;
2829 })
2830
2831 (define_expand "aarch64_uaddl2<mode>"
2832 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2833 (match_operand:VQW 1 "register_operand" "w")
2834 (match_operand:VQW 2 "register_operand" "w")]
2835 "TARGET_SIMD"
2836 {
2837 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2838 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2839 operands[2], p));
2840 DONE;
2841 })
2842
2843 (define_expand "aarch64_ssubl2<mode>"
2844 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2845 (match_operand:VQW 1 "register_operand" "w")
2846 (match_operand:VQW 2 "register_operand" "w")]
2847 "TARGET_SIMD"
2848 {
2849 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2850 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2851 operands[2], p));
2852 DONE;
2853 })
2854
2855 (define_expand "aarch64_usubl2<mode>"
2856 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2857 (match_operand:VQW 1 "register_operand" "w")
2858 (match_operand:VQW 2 "register_operand" "w")]
2859 "TARGET_SIMD"
2860 {
2861 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2862 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2863 operands[2], p));
2864 DONE;
2865 })
2866
2867 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2868 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2869 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2870 (match_operand:VD_BHSI 1 "register_operand" "w"))
2871 (ANY_EXTEND:<VWIDE>
2872 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2873 "TARGET_SIMD"
2874 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2875 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2876 )
2877
2878 ;; <su><addsub>w<q>.
2879
2880 (define_expand "widen_ssum<mode>3"
2881 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2882 (plus:<VDBLW> (sign_extend:<VDBLW>
2883 (match_operand:VQW 1 "register_operand" ""))
2884 (match_operand:<VDBLW> 2 "register_operand" "")))]
2885 "TARGET_SIMD"
2886 {
2887 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2888 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2889
2890 emit_insn (gen_aarch64_saddw<mode>_internal (temp, operands[2],
2891 operands[1], p));
2892 emit_insn (gen_aarch64_saddw2<mode> (operands[0], temp, operands[1]));
2893 DONE;
2894 }
2895 )
2896
2897 (define_expand "widen_ssum<mode>3"
2898 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2899 (plus:<VWIDE> (sign_extend:<VWIDE>
2900 (match_operand:VD_BHSI 1 "register_operand" ""))
2901 (match_operand:<VWIDE> 2 "register_operand" "")))]
2902 "TARGET_SIMD"
2903 {
2904 emit_insn (gen_aarch64_saddw<mode> (operands[0], operands[2], operands[1]));
2905 DONE;
2906 })
2907
2908 (define_expand "widen_usum<mode>3"
2909 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2910 (plus:<VDBLW> (zero_extend:<VDBLW>
2911 (match_operand:VQW 1 "register_operand" ""))
2912 (match_operand:<VDBLW> 2 "register_operand" "")))]
2913 "TARGET_SIMD"
2914 {
2915 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2916 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2917
2918 emit_insn (gen_aarch64_uaddw<mode>_internal (temp, operands[2],
2919 operands[1], p));
2920 emit_insn (gen_aarch64_uaddw2<mode> (operands[0], temp, operands[1]));
2921 DONE;
2922 }
2923 )
2924
2925 (define_expand "widen_usum<mode>3"
2926 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2927 (plus:<VWIDE> (zero_extend:<VWIDE>
2928 (match_operand:VD_BHSI 1 "register_operand" ""))
2929 (match_operand:<VWIDE> 2 "register_operand" "")))]
2930 "TARGET_SIMD"
2931 {
2932 emit_insn (gen_aarch64_uaddw<mode> (operands[0], operands[2], operands[1]));
2933 DONE;
2934 })
2935
2936 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2937 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2938 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2939 (ANY_EXTEND:<VWIDE>
2940 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2941 "TARGET_SIMD"
2942 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2943 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2944 )
2945
2946 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>_internal"
2947 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2948 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2949 (ANY_EXTEND:<VWIDE>
2950 (vec_select:<VHALF>
2951 (match_operand:VQW 2 "register_operand" "w")
2952 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))))]
2953 "TARGET_SIMD"
2954 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
2955 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2956 )
2957
2958 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2959 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2960 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2961 (ANY_EXTEND:<VWIDE>
2962 (vec_select:<VHALF>
2963 (match_operand:VQW 2 "register_operand" "w")
2964 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2965 "TARGET_SIMD"
2966 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2967 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2968 )
2969
2970 (define_expand "aarch64_saddw2<mode>"
2971 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2972 (match_operand:<VWIDE> 1 "register_operand" "w")
2973 (match_operand:VQW 2 "register_operand" "w")]
2974 "TARGET_SIMD"
2975 {
2976 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2977 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2978 operands[2], p));
2979 DONE;
2980 })
2981
2982 (define_expand "aarch64_uaddw2<mode>"
2983 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2984 (match_operand:<VWIDE> 1 "register_operand" "w")
2985 (match_operand:VQW 2 "register_operand" "w")]
2986 "TARGET_SIMD"
2987 {
2988 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2989 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2990 operands[2], p));
2991 DONE;
2992 })
2993
2994
2995 (define_expand "aarch64_ssubw2<mode>"
2996 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2997 (match_operand:<VWIDE> 1 "register_operand" "w")
2998 (match_operand:VQW 2 "register_operand" "w")]
2999 "TARGET_SIMD"
3000 {
3001 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3002 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
3003 operands[2], p));
3004 DONE;
3005 })
3006
3007 (define_expand "aarch64_usubw2<mode>"
3008 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3009 (match_operand:<VWIDE> 1 "register_operand" "w")
3010 (match_operand:VQW 2 "register_operand" "w")]
3011 "TARGET_SIMD"
3012 {
3013 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3014 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
3015 operands[2], p));
3016 DONE;
3017 })
3018
3019 ;; <su><r>h<addsub>.
3020
3021 (define_insn "aarch64_<sur>h<addsub><mode>"
3022 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
3023 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
3024 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
3025 HADDSUB))]
3026 "TARGET_SIMD"
3027 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3028 [(set_attr "type" "neon_<addsub>_halve<q>")]
3029 )
3030
3031 ;; <r><addsub>hn<q>.
3032
3033 (define_insn "aarch64_<sur><addsub>hn<mode>"
3034 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3035 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
3036 (match_operand:VQN 2 "register_operand" "w")]
3037 ADDSUBHN))]
3038 "TARGET_SIMD"
3039 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
3040 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3041 )
3042
3043 (define_insn "aarch64_<sur><addsub>hn2<mode>"
3044 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
3045 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
3046 (match_operand:VQN 2 "register_operand" "w")
3047 (match_operand:VQN 3 "register_operand" "w")]
3048 ADDSUBHN2))]
3049 "TARGET_SIMD"
3050 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
3051 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3052 )
3053
3054 ;; pmul.
3055
3056 (define_insn "aarch64_pmul<mode>"
3057 [(set (match_operand:VB 0 "register_operand" "=w")
3058 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
3059 (match_operand:VB 2 "register_operand" "w")]
3060 UNSPEC_PMUL))]
3061 "TARGET_SIMD"
3062 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3063 [(set_attr "type" "neon_mul_<Vetype><q>")]
3064 )
3065
3066 ;; fmulx.
3067
3068 (define_insn "aarch64_fmulx<mode>"
3069 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
3070 (unspec:VHSDF_HSDF
3071 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
3072 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
3073 UNSPEC_FMULX))]
3074 "TARGET_SIMD"
3075 "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3076 [(set_attr "type" "neon_fp_mul_<stype>")]
3077 )
3078
3079 ;; vmulxq_lane_f32, and vmulx_laneq_f32
3080
3081 (define_insn "*aarch64_mulx_elt_<vswap_width_name><mode>"
3082 [(set (match_operand:VDQSF 0 "register_operand" "=w")
3083 (unspec:VDQSF
3084 [(match_operand:VDQSF 1 "register_operand" "w")
3085 (vec_duplicate:VDQSF
3086 (vec_select:<VEL>
3087 (match_operand:<VSWAP_WIDTH> 2 "register_operand" "w")
3088 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3089 UNSPEC_FMULX))]
3090 "TARGET_SIMD"
3091 {
3092 operands[3] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
3093 INTVAL (operands[3])));
3094 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3095 }
3096 [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
3097 )
3098
3099 ;; vmulxq_laneq_f32, vmulxq_laneq_f64, vmulx_lane_f32
3100
3101 (define_insn "*aarch64_mulx_elt<mode>"
3102 [(set (match_operand:VDQF 0 "register_operand" "=w")
3103 (unspec:VDQF
3104 [(match_operand:VDQF 1 "register_operand" "w")
3105 (vec_duplicate:VDQF
3106 (vec_select:<VEL>
3107 (match_operand:VDQF 2 "register_operand" "w")
3108 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3109 UNSPEC_FMULX))]
3110 "TARGET_SIMD"
3111 {
3112 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3113 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3114 }
3115 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
3116 )
3117
3118 ;; vmulxq_lane
3119
3120 (define_insn "*aarch64_mulx_elt_from_dup<mode>"
3121 [(set (match_operand:VHSDF 0 "register_operand" "=w")
3122 (unspec:VHSDF
3123 [(match_operand:VHSDF 1 "register_operand" "w")
3124 (vec_duplicate:VHSDF
3125 (match_operand:<VEL> 2 "register_operand" "w"))]
3126 UNSPEC_FMULX))]
3127 "TARGET_SIMD"
3128 "fmulx\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[0]";
3129 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
3130 )
3131
3132 ;; vmulxs_lane_f32, vmulxs_laneq_f32
3133 ;; vmulxd_lane_f64 == vmulx_lane_f64
3134 ;; vmulxd_laneq_f64 == vmulx_laneq_f64
3135
3136 (define_insn "*aarch64_vgetfmulx<mode>"
3137 [(set (match_operand:<VEL> 0 "register_operand" "=w")
3138 (unspec:<VEL>
3139 [(match_operand:<VEL> 1 "register_operand" "w")
3140 (vec_select:<VEL>
3141 (match_operand:VDQF_DF 2 "register_operand" "w")
3142 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3143 UNSPEC_FMULX))]
3144 "TARGET_SIMD"
3145 {
3146 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3147 return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
3148 }
3149 [(set_attr "type" "fmul<Vetype>")]
3150 )
3151 ;; <su>q<addsub>
3152
3153 (define_insn "aarch64_<su_optab><optab><mode>"
3154 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3155 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
3156 (match_operand:VSDQ_I 2 "register_operand" "w")))]
3157 "TARGET_SIMD"
3158 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3159 [(set_attr "type" "neon_<optab><q>")]
3160 )
3161
3162 ;; suqadd and usqadd
3163
3164 (define_insn "aarch64_<sur>qadd<mode>"
3165 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3166 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
3167 (match_operand:VSDQ_I 2 "register_operand" "w")]
3168 USSUQADD))]
3169 "TARGET_SIMD"
3170 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
3171 [(set_attr "type" "neon_qadd<q>")]
3172 )
3173
3174 ;; sqmovun
3175
3176 (define_insn "aarch64_sqmovun<mode>"
3177 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3178 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3179 UNSPEC_SQXTUN))]
3180 "TARGET_SIMD"
3181 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3182 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3183 )
3184
3185 ;; sqmovn and uqmovn
3186
3187 (define_insn "aarch64_<sur>qmovn<mode>"
3188 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3189 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3190 SUQMOVN))]
3191 "TARGET_SIMD"
3192 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3193 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3194 )
3195
3196 ;; <su>q<absneg>
3197
3198 (define_insn "aarch64_s<optab><mode>"
3199 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3200 (UNQOPS:VSDQ_I
3201 (match_operand:VSDQ_I 1 "register_operand" "w")))]
3202 "TARGET_SIMD"
3203 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
3204 [(set_attr "type" "neon_<optab><q>")]
3205 )
3206
3207 ;; sq<r>dmulh.
3208
3209 (define_insn "aarch64_sq<r>dmulh<mode>"
3210 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3211 (unspec:VSDQ_HSI
3212 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
3213 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
3214 VQDMULH))]
3215 "TARGET_SIMD"
3216 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3217 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
3218 )
3219
3220 ;; sq<r>dmulh_lane
3221
3222 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3223 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3224 (unspec:VDQHS
3225 [(match_operand:VDQHS 1 "register_operand" "w")
3226 (vec_select:<VEL>
3227 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3228 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3229 VQDMULH))]
3230 "TARGET_SIMD"
3231 "*
3232 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3233 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3234 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3235 )
3236
3237 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3238 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3239 (unspec:VDQHS
3240 [(match_operand:VDQHS 1 "register_operand" "w")
3241 (vec_select:<VEL>
3242 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3243 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3244 VQDMULH))]
3245 "TARGET_SIMD"
3246 "*
3247 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3248 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3249 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3250 )
3251
3252 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3253 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3254 (unspec:SD_HSI
3255 [(match_operand:SD_HSI 1 "register_operand" "w")
3256 (vec_select:<VEL>
3257 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3258 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3259 VQDMULH))]
3260 "TARGET_SIMD"
3261 "*
3262 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3263 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3264 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3265 )
3266
3267 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3268 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3269 (unspec:SD_HSI
3270 [(match_operand:SD_HSI 1 "register_operand" "w")
3271 (vec_select:<VEL>
3272 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3273 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3274 VQDMULH))]
3275 "TARGET_SIMD"
3276 "*
3277 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3278 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3279 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3280 )
3281
3282 ;; sqrdml[as]h.
3283
3284 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>"
3285 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3286 (unspec:VSDQ_HSI
3287 [(match_operand:VSDQ_HSI 1 "register_operand" "0")
3288 (match_operand:VSDQ_HSI 2 "register_operand" "w")
3289 (match_operand:VSDQ_HSI 3 "register_operand" "w")]
3290 SQRDMLH_AS))]
3291 "TARGET_SIMD_RDMA"
3292 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3293 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3294 )
3295
3296 ;; sqrdml[as]h_lane.
3297
3298 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3299 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3300 (unspec:VDQHS
3301 [(match_operand:VDQHS 1 "register_operand" "0")
3302 (match_operand:VDQHS 2 "register_operand" "w")
3303 (vec_select:<VEL>
3304 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3305 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3306 SQRDMLH_AS))]
3307 "TARGET_SIMD_RDMA"
3308 {
3309 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3310 return
3311 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3312 }
3313 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3314 )
3315
3316 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3317 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3318 (unspec:SD_HSI
3319 [(match_operand:SD_HSI 1 "register_operand" "0")
3320 (match_operand:SD_HSI 2 "register_operand" "w")
3321 (vec_select:<VEL>
3322 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3323 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3324 SQRDMLH_AS))]
3325 "TARGET_SIMD_RDMA"
3326 {
3327 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3328 return
3329 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
3330 }
3331 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3332 )
3333
3334 ;; sqrdml[as]h_laneq.
3335
3336 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3337 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3338 (unspec:VDQHS
3339 [(match_operand:VDQHS 1 "register_operand" "0")
3340 (match_operand:VDQHS 2 "register_operand" "w")
3341 (vec_select:<VEL>
3342 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3343 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3344 SQRDMLH_AS))]
3345 "TARGET_SIMD_RDMA"
3346 {
3347 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3348 return
3349 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3350 }
3351 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3352 )
3353
3354 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3355 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3356 (unspec:SD_HSI
3357 [(match_operand:SD_HSI 1 "register_operand" "0")
3358 (match_operand:SD_HSI 2 "register_operand" "w")
3359 (vec_select:<VEL>
3360 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3361 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3362 SQRDMLH_AS))]
3363 "TARGET_SIMD_RDMA"
3364 {
3365 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3366 return
3367 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
3368 }
3369 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3370 )
3371
3372 ;; vqdml[sa]l
3373
3374 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
3375 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3376 (SBINQOPS:<VWIDE>
3377 (match_operand:<VWIDE> 1 "register_operand" "0")
3378 (ss_ashift:<VWIDE>
3379 (mult:<VWIDE>
3380 (sign_extend:<VWIDE>
3381 (match_operand:VSD_HSI 2 "register_operand" "w"))
3382 (sign_extend:<VWIDE>
3383 (match_operand:VSD_HSI 3 "register_operand" "w")))
3384 (const_int 1))))]
3385 "TARGET_SIMD"
3386 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3387 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3388 )
3389
3390 ;; vqdml[sa]l_lane
3391
3392 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3393 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3394 (SBINQOPS:<VWIDE>
3395 (match_operand:<VWIDE> 1 "register_operand" "0")
3396 (ss_ashift:<VWIDE>
3397 (mult:<VWIDE>
3398 (sign_extend:<VWIDE>
3399 (match_operand:VD_HSI 2 "register_operand" "w"))
3400 (sign_extend:<VWIDE>
3401 (vec_duplicate:VD_HSI
3402 (vec_select:<VEL>
3403 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3404 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3405 ))
3406 (const_int 1))))]
3407 "TARGET_SIMD"
3408 {
3409 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3410 return
3411 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3412 }
3413 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3414 )
3415
3416 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3417 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3418 (SBINQOPS:<VWIDE>
3419 (match_operand:<VWIDE> 1 "register_operand" "0")
3420 (ss_ashift:<VWIDE>
3421 (mult:<VWIDE>
3422 (sign_extend:<VWIDE>
3423 (match_operand:VD_HSI 2 "register_operand" "w"))
3424 (sign_extend:<VWIDE>
3425 (vec_duplicate:VD_HSI
3426 (vec_select:<VEL>
3427 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3428 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3429 ))
3430 (const_int 1))))]
3431 "TARGET_SIMD"
3432 {
3433 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3434 return
3435 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3436 }
3437 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3438 )
3439
3440 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3441 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3442 (SBINQOPS:<VWIDE>
3443 (match_operand:<VWIDE> 1 "register_operand" "0")
3444 (ss_ashift:<VWIDE>
3445 (mult:<VWIDE>
3446 (sign_extend:<VWIDE>
3447 (match_operand:SD_HSI 2 "register_operand" "w"))
3448 (sign_extend:<VWIDE>
3449 (vec_select:<VEL>
3450 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3451 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3452 )
3453 (const_int 1))))]
3454 "TARGET_SIMD"
3455 {
3456 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3457 return
3458 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3459 }
3460 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3461 )
3462
3463 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3464 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3465 (SBINQOPS:<VWIDE>
3466 (match_operand:<VWIDE> 1 "register_operand" "0")
3467 (ss_ashift:<VWIDE>
3468 (mult:<VWIDE>
3469 (sign_extend:<VWIDE>
3470 (match_operand:SD_HSI 2 "register_operand" "w"))
3471 (sign_extend:<VWIDE>
3472 (vec_select:<VEL>
3473 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3474 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3475 )
3476 (const_int 1))))]
3477 "TARGET_SIMD"
3478 {
3479 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3480 return
3481 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3482 }
3483 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3484 )
3485
3486 ;; vqdml[sa]l_n
3487
3488 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3489 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3490 (SBINQOPS:<VWIDE>
3491 (match_operand:<VWIDE> 1 "register_operand" "0")
3492 (ss_ashift:<VWIDE>
3493 (mult:<VWIDE>
3494 (sign_extend:<VWIDE>
3495 (match_operand:VD_HSI 2 "register_operand" "w"))
3496 (sign_extend:<VWIDE>
3497 (vec_duplicate:VD_HSI
3498 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3499 (const_int 1))))]
3500 "TARGET_SIMD"
3501 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3502 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3503 )
3504
3505 ;; sqdml[as]l2
3506
3507 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3508 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3509 (SBINQOPS:<VWIDE>
3510 (match_operand:<VWIDE> 1 "register_operand" "0")
3511 (ss_ashift:<VWIDE>
3512 (mult:<VWIDE>
3513 (sign_extend:<VWIDE>
3514 (vec_select:<VHALF>
3515 (match_operand:VQ_HSI 2 "register_operand" "w")
3516 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3517 (sign_extend:<VWIDE>
3518 (vec_select:<VHALF>
3519 (match_operand:VQ_HSI 3 "register_operand" "w")
3520 (match_dup 4))))
3521 (const_int 1))))]
3522 "TARGET_SIMD"
3523 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3524 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3525 )
3526
3527 (define_expand "aarch64_sqdmlal2<mode>"
3528 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3529 (match_operand:<VWIDE> 1 "register_operand" "w")
3530 (match_operand:VQ_HSI 2 "register_operand" "w")
3531 (match_operand:VQ_HSI 3 "register_operand" "w")]
3532 "TARGET_SIMD"
3533 {
3534 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3535 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3536 operands[2], operands[3], p));
3537 DONE;
3538 })
3539
3540 (define_expand "aarch64_sqdmlsl2<mode>"
3541 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3542 (match_operand:<VWIDE> 1 "register_operand" "w")
3543 (match_operand:VQ_HSI 2 "register_operand" "w")
3544 (match_operand:VQ_HSI 3 "register_operand" "w")]
3545 "TARGET_SIMD"
3546 {
3547 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3548 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3549 operands[2], operands[3], p));
3550 DONE;
3551 })
3552
3553 ;; vqdml[sa]l2_lane
3554
3555 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3556 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3557 (SBINQOPS:<VWIDE>
3558 (match_operand:<VWIDE> 1 "register_operand" "0")
3559 (ss_ashift:<VWIDE>
3560 (mult:<VWIDE>
3561 (sign_extend:<VWIDE>
3562 (vec_select:<VHALF>
3563 (match_operand:VQ_HSI 2 "register_operand" "w")
3564 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3565 (sign_extend:<VWIDE>
3566 (vec_duplicate:<VHALF>
3567 (vec_select:<VEL>
3568 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3569 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3570 ))))
3571 (const_int 1))))]
3572 "TARGET_SIMD"
3573 {
3574 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3575 return
3576 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3577 }
3578 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3579 )
3580
3581 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3582 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3583 (SBINQOPS:<VWIDE>
3584 (match_operand:<VWIDE> 1 "register_operand" "0")
3585 (ss_ashift:<VWIDE>
3586 (mult:<VWIDE>
3587 (sign_extend:<VWIDE>
3588 (vec_select:<VHALF>
3589 (match_operand:VQ_HSI 2 "register_operand" "w")
3590 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3591 (sign_extend:<VWIDE>
3592 (vec_duplicate:<VHALF>
3593 (vec_select:<VEL>
3594 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3595 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3596 ))))
3597 (const_int 1))))]
3598 "TARGET_SIMD"
3599 {
3600 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3601 return
3602 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3603 }
3604 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3605 )
3606
3607 (define_expand "aarch64_sqdmlal2_lane<mode>"
3608 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3609 (match_operand:<VWIDE> 1 "register_operand" "w")
3610 (match_operand:VQ_HSI 2 "register_operand" "w")
3611 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3612 (match_operand:SI 4 "immediate_operand" "i")]
3613 "TARGET_SIMD"
3614 {
3615 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3616 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3617 operands[2], operands[3],
3618 operands[4], p));
3619 DONE;
3620 })
3621
3622 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3623 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3624 (match_operand:<VWIDE> 1 "register_operand" "w")
3625 (match_operand:VQ_HSI 2 "register_operand" "w")
3626 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3627 (match_operand:SI 4 "immediate_operand" "i")]
3628 "TARGET_SIMD"
3629 {
3630 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3631 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3632 operands[2], operands[3],
3633 operands[4], p));
3634 DONE;
3635 })
3636
3637 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3638 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3639 (match_operand:<VWIDE> 1 "register_operand" "w")
3640 (match_operand:VQ_HSI 2 "register_operand" "w")
3641 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3642 (match_operand:SI 4 "immediate_operand" "i")]
3643 "TARGET_SIMD"
3644 {
3645 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3646 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3647 operands[2], operands[3],
3648 operands[4], p));
3649 DONE;
3650 })
3651
3652 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3653 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3654 (match_operand:<VWIDE> 1 "register_operand" "w")
3655 (match_operand:VQ_HSI 2 "register_operand" "w")
3656 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3657 (match_operand:SI 4 "immediate_operand" "i")]
3658 "TARGET_SIMD"
3659 {
3660 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3661 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3662 operands[2], operands[3],
3663 operands[4], p));
3664 DONE;
3665 })
3666
3667 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3668 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3669 (SBINQOPS:<VWIDE>
3670 (match_operand:<VWIDE> 1 "register_operand" "0")
3671 (ss_ashift:<VWIDE>
3672 (mult:<VWIDE>
3673 (sign_extend:<VWIDE>
3674 (vec_select:<VHALF>
3675 (match_operand:VQ_HSI 2 "register_operand" "w")
3676 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3677 (sign_extend:<VWIDE>
3678 (vec_duplicate:<VHALF>
3679 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3680 (const_int 1))))]
3681 "TARGET_SIMD"
3682 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3683 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3684 )
3685
3686 (define_expand "aarch64_sqdmlal2_n<mode>"
3687 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3688 (match_operand:<VWIDE> 1 "register_operand" "w")
3689 (match_operand:VQ_HSI 2 "register_operand" "w")
3690 (match_operand:<VEL> 3 "register_operand" "w")]
3691 "TARGET_SIMD"
3692 {
3693 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3694 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3695 operands[2], operands[3],
3696 p));
3697 DONE;
3698 })
3699
3700 (define_expand "aarch64_sqdmlsl2_n<mode>"
3701 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3702 (match_operand:<VWIDE> 1 "register_operand" "w")
3703 (match_operand:VQ_HSI 2 "register_operand" "w")
3704 (match_operand:<VEL> 3 "register_operand" "w")]
3705 "TARGET_SIMD"
3706 {
3707 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3708 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3709 operands[2], operands[3],
3710 p));
3711 DONE;
3712 })
3713
3714 ;; vqdmull
3715
3716 (define_insn "aarch64_sqdmull<mode>"
3717 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3718 (ss_ashift:<VWIDE>
3719 (mult:<VWIDE>
3720 (sign_extend:<VWIDE>
3721 (match_operand:VSD_HSI 1 "register_operand" "w"))
3722 (sign_extend:<VWIDE>
3723 (match_operand:VSD_HSI 2 "register_operand" "w")))
3724 (const_int 1)))]
3725 "TARGET_SIMD"
3726 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3727 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3728 )
3729
3730 ;; vqdmull_lane
3731
3732 (define_insn "aarch64_sqdmull_lane<mode>"
3733 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3734 (ss_ashift:<VWIDE>
3735 (mult:<VWIDE>
3736 (sign_extend:<VWIDE>
3737 (match_operand:VD_HSI 1 "register_operand" "w"))
3738 (sign_extend:<VWIDE>
3739 (vec_duplicate:VD_HSI
3740 (vec_select:<VEL>
3741 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3742 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3743 ))
3744 (const_int 1)))]
3745 "TARGET_SIMD"
3746 {
3747 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3748 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3749 }
3750 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3751 )
3752
3753 (define_insn "aarch64_sqdmull_laneq<mode>"
3754 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3755 (ss_ashift:<VWIDE>
3756 (mult:<VWIDE>
3757 (sign_extend:<VWIDE>
3758 (match_operand:VD_HSI 1 "register_operand" "w"))
3759 (sign_extend:<VWIDE>
3760 (vec_duplicate:VD_HSI
3761 (vec_select:<VEL>
3762 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3763 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3764 ))
3765 (const_int 1)))]
3766 "TARGET_SIMD"
3767 {
3768 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3769 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3770 }
3771 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3772 )
3773
3774 (define_insn "aarch64_sqdmull_lane<mode>"
3775 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3776 (ss_ashift:<VWIDE>
3777 (mult:<VWIDE>
3778 (sign_extend:<VWIDE>
3779 (match_operand:SD_HSI 1 "register_operand" "w"))
3780 (sign_extend:<VWIDE>
3781 (vec_select:<VEL>
3782 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3783 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3784 ))
3785 (const_int 1)))]
3786 "TARGET_SIMD"
3787 {
3788 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3789 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3790 }
3791 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3792 )
3793
3794 (define_insn "aarch64_sqdmull_laneq<mode>"
3795 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3796 (ss_ashift:<VWIDE>
3797 (mult:<VWIDE>
3798 (sign_extend:<VWIDE>
3799 (match_operand:SD_HSI 1 "register_operand" "w"))
3800 (sign_extend:<VWIDE>
3801 (vec_select:<VEL>
3802 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3803 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3804 ))
3805 (const_int 1)))]
3806 "TARGET_SIMD"
3807 {
3808 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3809 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3810 }
3811 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3812 )
3813
3814 ;; vqdmull_n
3815
3816 (define_insn "aarch64_sqdmull_n<mode>"
3817 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3818 (ss_ashift:<VWIDE>
3819 (mult:<VWIDE>
3820 (sign_extend:<VWIDE>
3821 (match_operand:VD_HSI 1 "register_operand" "w"))
3822 (sign_extend:<VWIDE>
3823 (vec_duplicate:VD_HSI
3824 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3825 )
3826 (const_int 1)))]
3827 "TARGET_SIMD"
3828 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3829 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3830 )
3831
3832 ;; vqdmull2
3833
3834
3835
3836 (define_insn "aarch64_sqdmull2<mode>_internal"
3837 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3838 (ss_ashift:<VWIDE>
3839 (mult:<VWIDE>
3840 (sign_extend:<VWIDE>
3841 (vec_select:<VHALF>
3842 (match_operand:VQ_HSI 1 "register_operand" "w")
3843 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3844 (sign_extend:<VWIDE>
3845 (vec_select:<VHALF>
3846 (match_operand:VQ_HSI 2 "register_operand" "w")
3847 (match_dup 3)))
3848 )
3849 (const_int 1)))]
3850 "TARGET_SIMD"
3851 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3852 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3853 )
3854
3855 (define_expand "aarch64_sqdmull2<mode>"
3856 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3857 (match_operand:VQ_HSI 1 "register_operand" "w")
3858 (match_operand:VQ_HSI 2 "register_operand" "w")]
3859 "TARGET_SIMD"
3860 {
3861 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3862 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3863 operands[2], p));
3864 DONE;
3865 })
3866
3867 ;; vqdmull2_lane
3868
3869 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3870 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3871 (ss_ashift:<VWIDE>
3872 (mult:<VWIDE>
3873 (sign_extend:<VWIDE>
3874 (vec_select:<VHALF>
3875 (match_operand:VQ_HSI 1 "register_operand" "w")
3876 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3877 (sign_extend:<VWIDE>
3878 (vec_duplicate:<VHALF>
3879 (vec_select:<VEL>
3880 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3881 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3882 ))
3883 (const_int 1)))]
3884 "TARGET_SIMD"
3885 {
3886 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3887 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3888 }
3889 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3890 )
3891
3892 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3893 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3894 (ss_ashift:<VWIDE>
3895 (mult:<VWIDE>
3896 (sign_extend:<VWIDE>
3897 (vec_select:<VHALF>
3898 (match_operand:VQ_HSI 1 "register_operand" "w")
3899 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3900 (sign_extend:<VWIDE>
3901 (vec_duplicate:<VHALF>
3902 (vec_select:<VEL>
3903 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3904 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3905 ))
3906 (const_int 1)))]
3907 "TARGET_SIMD"
3908 {
3909 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3910 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3911 }
3912 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3913 )
3914
3915 (define_expand "aarch64_sqdmull2_lane<mode>"
3916 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3917 (match_operand:VQ_HSI 1 "register_operand" "w")
3918 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3919 (match_operand:SI 3 "immediate_operand" "i")]
3920 "TARGET_SIMD"
3921 {
3922 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3923 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3924 operands[2], operands[3],
3925 p));
3926 DONE;
3927 })
3928
3929 (define_expand "aarch64_sqdmull2_laneq<mode>"
3930 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3931 (match_operand:VQ_HSI 1 "register_operand" "w")
3932 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3933 (match_operand:SI 3 "immediate_operand" "i")]
3934 "TARGET_SIMD"
3935 {
3936 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3937 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3938 operands[2], operands[3],
3939 p));
3940 DONE;
3941 })
3942
3943 ;; vqdmull2_n
3944
3945 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3946 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3947 (ss_ashift:<VWIDE>
3948 (mult:<VWIDE>
3949 (sign_extend:<VWIDE>
3950 (vec_select:<VHALF>
3951 (match_operand:VQ_HSI 1 "register_operand" "w")
3952 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3953 (sign_extend:<VWIDE>
3954 (vec_duplicate:<VHALF>
3955 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3956 )
3957 (const_int 1)))]
3958 "TARGET_SIMD"
3959 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3960 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3961 )
3962
3963 (define_expand "aarch64_sqdmull2_n<mode>"
3964 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3965 (match_operand:VQ_HSI 1 "register_operand" "w")
3966 (match_operand:<VEL> 2 "register_operand" "w")]
3967 "TARGET_SIMD"
3968 {
3969 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3970 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3971 operands[2], p));
3972 DONE;
3973 })
3974
3975 ;; vshl
3976
3977 (define_insn "aarch64_<sur>shl<mode>"
3978 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3979 (unspec:VSDQ_I_DI
3980 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3981 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3982 VSHL))]
3983 "TARGET_SIMD"
3984 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3985 [(set_attr "type" "neon_shift_reg<q>")]
3986 )
3987
3988
3989 ;; vqshl
3990
3991 (define_insn "aarch64_<sur>q<r>shl<mode>"
3992 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3993 (unspec:VSDQ_I
3994 [(match_operand:VSDQ_I 1 "register_operand" "w")
3995 (match_operand:VSDQ_I 2 "register_operand" "w")]
3996 VQSHL))]
3997 "TARGET_SIMD"
3998 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3999 [(set_attr "type" "neon_sat_shift_reg<q>")]
4000 )
4001
4002 ;; vshll_n
4003
4004 (define_insn "aarch64_<sur>shll_n<mode>"
4005 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4006 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
4007 (match_operand:SI 2
4008 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
4009 VSHLL))]
4010 "TARGET_SIMD"
4011 {
4012 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4013 return "shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4014 else
4015 return "<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4016 }
4017 [(set_attr "type" "neon_shift_imm_long")]
4018 )
4019
4020 ;; vshll_high_n
4021
4022 (define_insn "aarch64_<sur>shll2_n<mode>"
4023 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4024 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
4025 (match_operand:SI 2 "immediate_operand" "i")]
4026 VSHLL))]
4027 "TARGET_SIMD"
4028 {
4029 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4030 return "shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4031 else
4032 return "<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4033 }
4034 [(set_attr "type" "neon_shift_imm_long")]
4035 )
4036
4037 ;; vrshr_n
4038
4039 (define_insn "aarch64_<sur>shr_n<mode>"
4040 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4041 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4042 (match_operand:SI 2
4043 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4044 VRSHR_N))]
4045 "TARGET_SIMD"
4046 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4047 [(set_attr "type" "neon_sat_shift_imm<q>")]
4048 )
4049
4050 ;; v(r)sra_n
4051
4052 (define_insn "aarch64_<sur>sra_n<mode>"
4053 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4054 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4055 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4056 (match_operand:SI 3
4057 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4058 VSRA))]
4059 "TARGET_SIMD"
4060 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4061 [(set_attr "type" "neon_shift_acc<q>")]
4062 )
4063
4064 ;; vs<lr>i_n
4065
4066 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
4067 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4068 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4069 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4070 (match_operand:SI 3
4071 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
4072 VSLRI))]
4073 "TARGET_SIMD"
4074 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4075 [(set_attr "type" "neon_shift_imm<q>")]
4076 )
4077
4078 ;; vqshl(u)
4079
4080 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
4081 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4082 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
4083 (match_operand:SI 2
4084 "aarch64_simd_shift_imm_<ve_mode>" "i")]
4085 VQSHL_N))]
4086 "TARGET_SIMD"
4087 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4088 [(set_attr "type" "neon_sat_shift_imm<q>")]
4089 )
4090
4091
4092 ;; vq(r)shr(u)n_n
4093
4094 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
4095 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
4096 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
4097 (match_operand:SI 2
4098 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4099 VQSHRN_N))]
4100 "TARGET_SIMD"
4101 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
4102 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
4103 )
4104
4105
4106 ;; cm(eq|ge|gt|lt|le)
4107 ;; Note, we have constraints for Dz and Z as different expanders
4108 ;; have different ideas of what should be passed to this pattern.
4109
4110 (define_insn "aarch64_cm<optab><mode>"
4111 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4112 (neg:<V_cmp_result>
4113 (COMPARISONS:<V_cmp_result>
4114 (match_operand:VDQ_I 1 "register_operand" "w,w")
4115 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4116 )))]
4117 "TARGET_SIMD"
4118 "@
4119 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4120 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
4121 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
4122 )
4123
4124 (define_insn_and_split "aarch64_cm<optab>di"
4125 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
4126 (neg:DI
4127 (COMPARISONS:DI
4128 (match_operand:DI 1 "register_operand" "w,w,r")
4129 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
4130 )))
4131 (clobber (reg:CC CC_REGNUM))]
4132 "TARGET_SIMD"
4133 "#"
4134 "reload_completed"
4135 [(set (match_operand:DI 0 "register_operand")
4136 (neg:DI
4137 (COMPARISONS:DI
4138 (match_operand:DI 1 "register_operand")
4139 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4140 )))]
4141 {
4142 /* If we are in the general purpose register file,
4143 we split to a sequence of comparison and store. */
4144 if (GP_REGNUM_P (REGNO (operands[0]))
4145 && GP_REGNUM_P (REGNO (operands[1])))
4146 {
4147 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
4148 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4149 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4150 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4151 DONE;
4152 }
4153 /* Otherwise, we expand to a similar pattern which does not
4154 clobber CC_REGNUM. */
4155 }
4156 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
4157 )
4158
4159 (define_insn "*aarch64_cm<optab>di"
4160 [(set (match_operand:DI 0 "register_operand" "=w,w")
4161 (neg:DI
4162 (COMPARISONS:DI
4163 (match_operand:DI 1 "register_operand" "w,w")
4164 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4165 )))]
4166 "TARGET_SIMD && reload_completed"
4167 "@
4168 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
4169 cm<optab>\t%d0, %d1, #0"
4170 [(set_attr "type" "neon_compare, neon_compare_zero")]
4171 )
4172
4173 ;; cm(hs|hi)
4174
4175 (define_insn "aarch64_cm<optab><mode>"
4176 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4177 (neg:<V_cmp_result>
4178 (UCOMPARISONS:<V_cmp_result>
4179 (match_operand:VDQ_I 1 "register_operand" "w")
4180 (match_operand:VDQ_I 2 "register_operand" "w")
4181 )))]
4182 "TARGET_SIMD"
4183 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4184 [(set_attr "type" "neon_compare<q>")]
4185 )
4186
4187 (define_insn_and_split "aarch64_cm<optab>di"
4188 [(set (match_operand:DI 0 "register_operand" "=w,r")
4189 (neg:DI
4190 (UCOMPARISONS:DI
4191 (match_operand:DI 1 "register_operand" "w,r")
4192 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
4193 )))
4194 (clobber (reg:CC CC_REGNUM))]
4195 "TARGET_SIMD"
4196 "#"
4197 "reload_completed"
4198 [(set (match_operand:DI 0 "register_operand")
4199 (neg:DI
4200 (UCOMPARISONS:DI
4201 (match_operand:DI 1 "register_operand")
4202 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4203 )))]
4204 {
4205 /* If we are in the general purpose register file,
4206 we split to a sequence of comparison and store. */
4207 if (GP_REGNUM_P (REGNO (operands[0]))
4208 && GP_REGNUM_P (REGNO (operands[1])))
4209 {
4210 machine_mode mode = CCmode;
4211 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4212 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4213 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4214 DONE;
4215 }
4216 /* Otherwise, we expand to a similar pattern which does not
4217 clobber CC_REGNUM. */
4218 }
4219 [(set_attr "type" "neon_compare,multiple")]
4220 )
4221
4222 (define_insn "*aarch64_cm<optab>di"
4223 [(set (match_operand:DI 0 "register_operand" "=w")
4224 (neg:DI
4225 (UCOMPARISONS:DI
4226 (match_operand:DI 1 "register_operand" "w")
4227 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
4228 )))]
4229 "TARGET_SIMD && reload_completed"
4230 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
4231 [(set_attr "type" "neon_compare")]
4232 )
4233
4234 ;; cmtst
4235
4236 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
4237 ;; we don't have any insns using ne, and aarch64_vcond outputs
4238 ;; not (neg (eq (and x y) 0))
4239 ;; which is rewritten by simplify_rtx as
4240 ;; plus (eq (and x y) 0) -1.
4241
4242 (define_insn "aarch64_cmtst<mode>"
4243 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4244 (plus:<V_cmp_result>
4245 (eq:<V_cmp_result>
4246 (and:VDQ_I
4247 (match_operand:VDQ_I 1 "register_operand" "w")
4248 (match_operand:VDQ_I 2 "register_operand" "w"))
4249 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
4250 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
4251 ]
4252 "TARGET_SIMD"
4253 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4254 [(set_attr "type" "neon_tst<q>")]
4255 )
4256
4257 (define_insn_and_split "aarch64_cmtstdi"
4258 [(set (match_operand:DI 0 "register_operand" "=w,r")
4259 (neg:DI
4260 (ne:DI
4261 (and:DI
4262 (match_operand:DI 1 "register_operand" "w,r")
4263 (match_operand:DI 2 "register_operand" "w,r"))
4264 (const_int 0))))
4265 (clobber (reg:CC CC_REGNUM))]
4266 "TARGET_SIMD"
4267 "#"
4268 "reload_completed"
4269 [(set (match_operand:DI 0 "register_operand")
4270 (neg:DI
4271 (ne:DI
4272 (and:DI
4273 (match_operand:DI 1 "register_operand")
4274 (match_operand:DI 2 "register_operand"))
4275 (const_int 0))))]
4276 {
4277 /* If we are in the general purpose register file,
4278 we split to a sequence of comparison and store. */
4279 if (GP_REGNUM_P (REGNO (operands[0]))
4280 && GP_REGNUM_P (REGNO (operands[1])))
4281 {
4282 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4283 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4284 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4285 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4286 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4287 DONE;
4288 }
4289 /* Otherwise, we expand to a similar pattern which does not
4290 clobber CC_REGNUM. */
4291 }
4292 [(set_attr "type" "neon_tst,multiple")]
4293 )
4294
4295 (define_insn "*aarch64_cmtstdi"
4296 [(set (match_operand:DI 0 "register_operand" "=w")
4297 (neg:DI
4298 (ne:DI
4299 (and:DI
4300 (match_operand:DI 1 "register_operand" "w")
4301 (match_operand:DI 2 "register_operand" "w"))
4302 (const_int 0))))]
4303 "TARGET_SIMD"
4304 "cmtst\t%d0, %d1, %d2"
4305 [(set_attr "type" "neon_tst")]
4306 )
4307
4308 ;; fcm(eq|ge|gt|le|lt)
4309
4310 (define_insn "aarch64_cm<optab><mode>"
4311 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4312 (neg:<V_cmp_result>
4313 (COMPARISONS:<V_cmp_result>
4314 (match_operand:VHSDF_HSDF 1 "register_operand" "w,w")
4315 (match_operand:VHSDF_HSDF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4316 )))]
4317 "TARGET_SIMD"
4318 "@
4319 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4320 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4321 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4322 )
4323
4324 ;; fac(ge|gt)
4325 ;; Note we can also handle what would be fac(le|lt) by
4326 ;; generating fac(ge|gt).
4327
4328 (define_insn "aarch64_fac<optab><mode>"
4329 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4330 (neg:<V_cmp_result>
4331 (FAC_COMPARISONS:<V_cmp_result>
4332 (abs:VHSDF_HSDF
4333 (match_operand:VHSDF_HSDF 1 "register_operand" "w"))
4334 (abs:VHSDF_HSDF
4335 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))
4336 )))]
4337 "TARGET_SIMD"
4338 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4339 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4340 )
4341
4342 ;; addp
4343
4344 (define_insn "aarch64_addp<mode>"
4345 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4346 (unspec:VD_BHSI
4347 [(match_operand:VD_BHSI 1 "register_operand" "w")
4348 (match_operand:VD_BHSI 2 "register_operand" "w")]
4349 UNSPEC_ADDP))]
4350 "TARGET_SIMD"
4351 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4352 [(set_attr "type" "neon_reduc_add<q>")]
4353 )
4354
4355 (define_insn "aarch64_addpdi"
4356 [(set (match_operand:DI 0 "register_operand" "=w")
4357 (unspec:DI
4358 [(match_operand:V2DI 1 "register_operand" "w")]
4359 UNSPEC_ADDP))]
4360 "TARGET_SIMD"
4361 "addp\t%d0, %1.2d"
4362 [(set_attr "type" "neon_reduc_add")]
4363 )
4364
4365 ;; sqrt
4366
4367 (define_expand "sqrt<mode>2"
4368 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4369 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4370 "TARGET_SIMD"
4371 {
4372 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
4373 DONE;
4374 })
4375
4376 (define_insn "*sqrt<mode>2"
4377 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4378 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4379 "TARGET_SIMD"
4380 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4381 [(set_attr "type" "neon_fp_sqrt_<stype><q>")]
4382 )
4383
4384 ;; Patterns for vector struct loads and stores.
4385
4386 (define_insn "aarch64_simd_ld2<mode>"
4387 [(set (match_operand:OI 0 "register_operand" "=w")
4388 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4389 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4390 UNSPEC_LD2))]
4391 "TARGET_SIMD"
4392 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4393 [(set_attr "type" "neon_load2_2reg<q>")]
4394 )
4395
4396 (define_insn "aarch64_simd_ld2r<mode>"
4397 [(set (match_operand:OI 0 "register_operand" "=w")
4398 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4399 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4400 UNSPEC_LD2_DUP))]
4401 "TARGET_SIMD"
4402 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4403 [(set_attr "type" "neon_load2_all_lanes<q>")]
4404 )
4405
4406 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4407 [(set (match_operand:OI 0 "register_operand" "=w")
4408 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4409 (match_operand:OI 2 "register_operand" "0")
4410 (match_operand:SI 3 "immediate_operand" "i")
4411 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4412 UNSPEC_LD2_LANE))]
4413 "TARGET_SIMD"
4414 {
4415 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4416 return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
4417 }
4418 [(set_attr "type" "neon_load2_one_lane")]
4419 )
4420
4421 (define_expand "vec_load_lanesoi<mode>"
4422 [(set (match_operand:OI 0 "register_operand" "=w")
4423 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4424 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4425 UNSPEC_LD2))]
4426 "TARGET_SIMD"
4427 {
4428 if (BYTES_BIG_ENDIAN)
4429 {
4430 rtx tmp = gen_reg_rtx (OImode);
4431 rtx mask = aarch64_reverse_mask (<MODE>mode);
4432 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
4433 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
4434 }
4435 else
4436 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
4437 DONE;
4438 })
4439
4440 (define_insn "aarch64_simd_st2<mode>"
4441 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4442 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4443 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4444 UNSPEC_ST2))]
4445 "TARGET_SIMD"
4446 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4447 [(set_attr "type" "neon_store2_2reg<q>")]
4448 )
4449
4450 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4451 (define_insn "aarch64_vec_store_lanesoi_lane<mode>"
4452 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4453 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
4454 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4455 (match_operand:SI 2 "immediate_operand" "i")]
4456 UNSPEC_ST2_LANE))]
4457 "TARGET_SIMD"
4458 {
4459 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4460 return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
4461 }
4462 [(set_attr "type" "neon_store2_one_lane<q>")]
4463 )
4464
4465 (define_expand "vec_store_lanesoi<mode>"
4466 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4467 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4468 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4469 UNSPEC_ST2))]
4470 "TARGET_SIMD"
4471 {
4472 if (BYTES_BIG_ENDIAN)
4473 {
4474 rtx tmp = gen_reg_rtx (OImode);
4475 rtx mask = aarch64_reverse_mask (<MODE>mode);
4476 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
4477 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
4478 }
4479 else
4480 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
4481 DONE;
4482 })
4483
4484 (define_insn "aarch64_simd_ld3<mode>"
4485 [(set (match_operand:CI 0 "register_operand" "=w")
4486 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4487 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4488 UNSPEC_LD3))]
4489 "TARGET_SIMD"
4490 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4491 [(set_attr "type" "neon_load3_3reg<q>")]
4492 )
4493
4494 (define_insn "aarch64_simd_ld3r<mode>"
4495 [(set (match_operand:CI 0 "register_operand" "=w")
4496 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4497 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4498 UNSPEC_LD3_DUP))]
4499 "TARGET_SIMD"
4500 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4501 [(set_attr "type" "neon_load3_all_lanes<q>")]
4502 )
4503
4504 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4505 [(set (match_operand:CI 0 "register_operand" "=w")
4506 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4507 (match_operand:CI 2 "register_operand" "0")
4508 (match_operand:SI 3 "immediate_operand" "i")
4509 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4510 UNSPEC_LD3_LANE))]
4511 "TARGET_SIMD"
4512 {
4513 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4514 return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
4515 }
4516 [(set_attr "type" "neon_load3_one_lane")]
4517 )
4518
4519 (define_expand "vec_load_lanesci<mode>"
4520 [(set (match_operand:CI 0 "register_operand" "=w")
4521 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4522 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4523 UNSPEC_LD3))]
4524 "TARGET_SIMD"
4525 {
4526 if (BYTES_BIG_ENDIAN)
4527 {
4528 rtx tmp = gen_reg_rtx (CImode);
4529 rtx mask = aarch64_reverse_mask (<MODE>mode);
4530 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4531 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4532 }
4533 else
4534 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4535 DONE;
4536 })
4537
4538 (define_insn "aarch64_simd_st3<mode>"
4539 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4540 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4541 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4542 UNSPEC_ST3))]
4543 "TARGET_SIMD"
4544 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4545 [(set_attr "type" "neon_store3_3reg<q>")]
4546 )
4547
4548 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4549 (define_insn "aarch64_vec_store_lanesci_lane<mode>"
4550 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4551 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
4552 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4553 (match_operand:SI 2 "immediate_operand" "i")]
4554 UNSPEC_ST3_LANE))]
4555 "TARGET_SIMD"
4556 {
4557 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4558 return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
4559 }
4560 [(set_attr "type" "neon_store3_one_lane<q>")]
4561 )
4562
4563 (define_expand "vec_store_lanesci<mode>"
4564 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4565 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4566 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4567 UNSPEC_ST3))]
4568 "TARGET_SIMD"
4569 {
4570 if (BYTES_BIG_ENDIAN)
4571 {
4572 rtx tmp = gen_reg_rtx (CImode);
4573 rtx mask = aarch64_reverse_mask (<MODE>mode);
4574 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4575 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4576 }
4577 else
4578 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4579 DONE;
4580 })
4581
4582 (define_insn "aarch64_simd_ld4<mode>"
4583 [(set (match_operand:XI 0 "register_operand" "=w")
4584 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4585 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4586 UNSPEC_LD4))]
4587 "TARGET_SIMD"
4588 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4589 [(set_attr "type" "neon_load4_4reg<q>")]
4590 )
4591
4592 (define_insn "aarch64_simd_ld4r<mode>"
4593 [(set (match_operand:XI 0 "register_operand" "=w")
4594 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4595 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4596 UNSPEC_LD4_DUP))]
4597 "TARGET_SIMD"
4598 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4599 [(set_attr "type" "neon_load4_all_lanes<q>")]
4600 )
4601
4602 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4603 [(set (match_operand:XI 0 "register_operand" "=w")
4604 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4605 (match_operand:XI 2 "register_operand" "0")
4606 (match_operand:SI 3 "immediate_operand" "i")
4607 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4608 UNSPEC_LD4_LANE))]
4609 "TARGET_SIMD"
4610 {
4611 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4612 return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
4613 }
4614 [(set_attr "type" "neon_load4_one_lane")]
4615 )
4616
4617 (define_expand "vec_load_lanesxi<mode>"
4618 [(set (match_operand:XI 0 "register_operand" "=w")
4619 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4620 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4621 UNSPEC_LD4))]
4622 "TARGET_SIMD"
4623 {
4624 if (BYTES_BIG_ENDIAN)
4625 {
4626 rtx tmp = gen_reg_rtx (XImode);
4627 rtx mask = aarch64_reverse_mask (<MODE>mode);
4628 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4629 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4630 }
4631 else
4632 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4633 DONE;
4634 })
4635
4636 (define_insn "aarch64_simd_st4<mode>"
4637 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4638 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4639 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4640 UNSPEC_ST4))]
4641 "TARGET_SIMD"
4642 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4643 [(set_attr "type" "neon_store4_4reg<q>")]
4644 )
4645
4646 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4647 (define_insn "aarch64_vec_store_lanesxi_lane<mode>"
4648 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4649 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
4650 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4651 (match_operand:SI 2 "immediate_operand" "i")]
4652 UNSPEC_ST4_LANE))]
4653 "TARGET_SIMD"
4654 {
4655 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4656 return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
4657 }
4658 [(set_attr "type" "neon_store4_one_lane<q>")]
4659 )
4660
4661 (define_expand "vec_store_lanesxi<mode>"
4662 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4663 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4664 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4665 UNSPEC_ST4))]
4666 "TARGET_SIMD"
4667 {
4668 if (BYTES_BIG_ENDIAN)
4669 {
4670 rtx tmp = gen_reg_rtx (XImode);
4671 rtx mask = aarch64_reverse_mask (<MODE>mode);
4672 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4673 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4674 }
4675 else
4676 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4677 DONE;
4678 })
4679
4680 (define_insn_and_split "aarch64_rev_reglist<mode>"
4681 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4682 (unspec:VSTRUCT
4683 [(match_operand:VSTRUCT 1 "register_operand" "w")
4684 (match_operand:V16QI 2 "register_operand" "w")]
4685 UNSPEC_REV_REGLIST))]
4686 "TARGET_SIMD"
4687 "#"
4688 "&& reload_completed"
4689 [(const_int 0)]
4690 {
4691 int i;
4692 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4693 for (i = 0; i < nregs; i++)
4694 {
4695 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4696 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4697 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4698 }
4699 DONE;
4700 }
4701 [(set_attr "type" "neon_tbl1_q")
4702 (set_attr "length" "<insn_count>")]
4703 )
4704
4705 ;; Reload patterns for AdvSIMD register list operands.
4706
4707 (define_expand "mov<mode>"
4708 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4709 (match_operand:VSTRUCT 1 "general_operand" ""))]
4710 "TARGET_SIMD"
4711 {
4712 if (can_create_pseudo_p ())
4713 {
4714 if (GET_CODE (operands[0]) != REG)
4715 operands[1] = force_reg (<MODE>mode, operands[1]);
4716 }
4717 })
4718
4719 (define_insn "*aarch64_mov<mode>"
4720 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4721 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4722 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4723 && (register_operand (operands[0], <MODE>mode)
4724 || register_operand (operands[1], <MODE>mode))"
4725 "@
4726 #
4727 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4728 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4729 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4730 neon_load<nregs>_<nregs>reg_q")
4731 (set_attr "length" "<insn_count>,4,4")]
4732 )
4733
4734 (define_insn "aarch64_be_ld1<mode>"
4735 [(set (match_operand:VALLDI_F16 0 "register_operand" "=w")
4736 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1
4737 "aarch64_simd_struct_operand" "Utv")]
4738 UNSPEC_LD1))]
4739 "TARGET_SIMD"
4740 "ld1\\t{%0<Vmtype>}, %1"
4741 [(set_attr "type" "neon_load1_1reg<q>")]
4742 )
4743
4744 (define_insn "aarch64_be_st1<mode>"
4745 [(set (match_operand:VALLDI_F16 0 "aarch64_simd_struct_operand" "=Utv")
4746 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1 "register_operand" "w")]
4747 UNSPEC_ST1))]
4748 "TARGET_SIMD"
4749 "st1\\t{%1<Vmtype>}, %0"
4750 [(set_attr "type" "neon_store1_1reg<q>")]
4751 )
4752
4753 (define_insn "*aarch64_be_movoi"
4754 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4755 (match_operand:OI 1 "general_operand" " w,w,m"))]
4756 "TARGET_SIMD && BYTES_BIG_ENDIAN
4757 && (register_operand (operands[0], OImode)
4758 || register_operand (operands[1], OImode))"
4759 "@
4760 #
4761 stp\\t%q1, %R1, %0
4762 ldp\\t%q0, %R0, %1"
4763 [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
4764 (set_attr "length" "8,4,4")]
4765 )
4766
4767 (define_insn "*aarch64_be_movci"
4768 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4769 (match_operand:CI 1 "general_operand" " w,w,o"))]
4770 "TARGET_SIMD && BYTES_BIG_ENDIAN
4771 && (register_operand (operands[0], CImode)
4772 || register_operand (operands[1], CImode))"
4773 "#"
4774 [(set_attr "type" "multiple")
4775 (set_attr "length" "12,4,4")]
4776 )
4777
4778 (define_insn "*aarch64_be_movxi"
4779 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4780 (match_operand:XI 1 "general_operand" " w,w,o"))]
4781 "TARGET_SIMD && BYTES_BIG_ENDIAN
4782 && (register_operand (operands[0], XImode)
4783 || register_operand (operands[1], XImode))"
4784 "#"
4785 [(set_attr "type" "multiple")
4786 (set_attr "length" "16,4,4")]
4787 )
4788
4789 (define_split
4790 [(set (match_operand:OI 0 "register_operand")
4791 (match_operand:OI 1 "register_operand"))]
4792 "TARGET_SIMD && reload_completed"
4793 [(const_int 0)]
4794 {
4795 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4796 DONE;
4797 })
4798
4799 (define_split
4800 [(set (match_operand:CI 0 "nonimmediate_operand")
4801 (match_operand:CI 1 "general_operand"))]
4802 "TARGET_SIMD && reload_completed"
4803 [(const_int 0)]
4804 {
4805 if (register_operand (operands[0], CImode)
4806 && register_operand (operands[1], CImode))
4807 {
4808 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4809 DONE;
4810 }
4811 else if (BYTES_BIG_ENDIAN)
4812 {
4813 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4814 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4815 emit_move_insn (gen_lowpart (V16QImode,
4816 simplify_gen_subreg (TImode, operands[0],
4817 CImode, 32)),
4818 gen_lowpart (V16QImode,
4819 simplify_gen_subreg (TImode, operands[1],
4820 CImode, 32)));
4821 DONE;
4822 }
4823 else
4824 FAIL;
4825 })
4826
4827 (define_split
4828 [(set (match_operand:XI 0 "nonimmediate_operand")
4829 (match_operand:XI 1 "general_operand"))]
4830 "TARGET_SIMD && reload_completed"
4831 [(const_int 0)]
4832 {
4833 if (register_operand (operands[0], XImode)
4834 && register_operand (operands[1], XImode))
4835 {
4836 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4837 DONE;
4838 }
4839 else if (BYTES_BIG_ENDIAN)
4840 {
4841 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4842 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4843 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4844 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4845 DONE;
4846 }
4847 else
4848 FAIL;
4849 })
4850
4851 (define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
4852 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4853 (match_operand:DI 1 "register_operand" "w")
4854 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4855 "TARGET_SIMD"
4856 {
4857 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4858 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4859 * <VSTRUCT:nregs>);
4860
4861 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
4862 mem));
4863 DONE;
4864 })
4865
4866 (define_insn "aarch64_ld2<mode>_dreg"
4867 [(set (match_operand:OI 0 "register_operand" "=w")
4868 (subreg:OI
4869 (vec_concat:<VRL2>
4870 (vec_concat:<VDBL>
4871 (unspec:VD
4872 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4873 UNSPEC_LD2)
4874 (vec_duplicate:VD (const_int 0)))
4875 (vec_concat:<VDBL>
4876 (unspec:VD [(match_dup 1)]
4877 UNSPEC_LD2)
4878 (vec_duplicate:VD (const_int 0)))) 0))]
4879 "TARGET_SIMD"
4880 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4881 [(set_attr "type" "neon_load2_2reg<q>")]
4882 )
4883
4884 (define_insn "aarch64_ld2<mode>_dreg"
4885 [(set (match_operand:OI 0 "register_operand" "=w")
4886 (subreg:OI
4887 (vec_concat:<VRL2>
4888 (vec_concat:<VDBL>
4889 (unspec:DX
4890 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4891 UNSPEC_LD2)
4892 (const_int 0))
4893 (vec_concat:<VDBL>
4894 (unspec:DX [(match_dup 1)]
4895 UNSPEC_LD2)
4896 (const_int 0))) 0))]
4897 "TARGET_SIMD"
4898 "ld1\\t{%S0.1d - %T0.1d}, %1"
4899 [(set_attr "type" "neon_load1_2reg<q>")]
4900 )
4901
4902 (define_insn "aarch64_ld3<mode>_dreg"
4903 [(set (match_operand:CI 0 "register_operand" "=w")
4904 (subreg:CI
4905 (vec_concat:<VRL3>
4906 (vec_concat:<VRL2>
4907 (vec_concat:<VDBL>
4908 (unspec:VD
4909 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4910 UNSPEC_LD3)
4911 (vec_duplicate:VD (const_int 0)))
4912 (vec_concat:<VDBL>
4913 (unspec:VD [(match_dup 1)]
4914 UNSPEC_LD3)
4915 (vec_duplicate:VD (const_int 0))))
4916 (vec_concat:<VDBL>
4917 (unspec:VD [(match_dup 1)]
4918 UNSPEC_LD3)
4919 (vec_duplicate:VD (const_int 0)))) 0))]
4920 "TARGET_SIMD"
4921 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4922 [(set_attr "type" "neon_load3_3reg<q>")]
4923 )
4924
4925 (define_insn "aarch64_ld3<mode>_dreg"
4926 [(set (match_operand:CI 0 "register_operand" "=w")
4927 (subreg:CI
4928 (vec_concat:<VRL3>
4929 (vec_concat:<VRL2>
4930 (vec_concat:<VDBL>
4931 (unspec:DX
4932 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4933 UNSPEC_LD3)
4934 (const_int 0))
4935 (vec_concat:<VDBL>
4936 (unspec:DX [(match_dup 1)]
4937 UNSPEC_LD3)
4938 (const_int 0)))
4939 (vec_concat:<VDBL>
4940 (unspec:DX [(match_dup 1)]
4941 UNSPEC_LD3)
4942 (const_int 0))) 0))]
4943 "TARGET_SIMD"
4944 "ld1\\t{%S0.1d - %U0.1d}, %1"
4945 [(set_attr "type" "neon_load1_3reg<q>")]
4946 )
4947
4948 (define_insn "aarch64_ld4<mode>_dreg"
4949 [(set (match_operand:XI 0 "register_operand" "=w")
4950 (subreg:XI
4951 (vec_concat:<VRL4>
4952 (vec_concat:<VRL2>
4953 (vec_concat:<VDBL>
4954 (unspec:VD
4955 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4956 UNSPEC_LD4)
4957 (vec_duplicate:VD (const_int 0)))
4958 (vec_concat:<VDBL>
4959 (unspec:VD [(match_dup 1)]
4960 UNSPEC_LD4)
4961 (vec_duplicate:VD (const_int 0))))
4962 (vec_concat:<VRL2>
4963 (vec_concat:<VDBL>
4964 (unspec:VD [(match_dup 1)]
4965 UNSPEC_LD4)
4966 (vec_duplicate:VD (const_int 0)))
4967 (vec_concat:<VDBL>
4968 (unspec:VD [(match_dup 1)]
4969 UNSPEC_LD4)
4970 (vec_duplicate:VD (const_int 0))))) 0))]
4971 "TARGET_SIMD"
4972 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4973 [(set_attr "type" "neon_load4_4reg<q>")]
4974 )
4975
4976 (define_insn "aarch64_ld4<mode>_dreg"
4977 [(set (match_operand:XI 0 "register_operand" "=w")
4978 (subreg:XI
4979 (vec_concat:<VRL4>
4980 (vec_concat:<VRL2>
4981 (vec_concat:<VDBL>
4982 (unspec:DX
4983 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4984 UNSPEC_LD4)
4985 (const_int 0))
4986 (vec_concat:<VDBL>
4987 (unspec:DX [(match_dup 1)]
4988 UNSPEC_LD4)
4989 (const_int 0)))
4990 (vec_concat:<VRL2>
4991 (vec_concat:<VDBL>
4992 (unspec:DX [(match_dup 1)]
4993 UNSPEC_LD4)
4994 (const_int 0))
4995 (vec_concat:<VDBL>
4996 (unspec:DX [(match_dup 1)]
4997 UNSPEC_LD4)
4998 (const_int 0)))) 0))]
4999 "TARGET_SIMD"
5000 "ld1\\t{%S0.1d - %V0.1d}, %1"
5001 [(set_attr "type" "neon_load1_4reg<q>")]
5002 )
5003
5004 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
5005 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5006 (match_operand:DI 1 "register_operand" "r")
5007 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5008 "TARGET_SIMD"
5009 {
5010 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5011 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5012
5013 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
5014 DONE;
5015 })
5016
5017 (define_expand "aarch64_ld1<VALL_F16:mode>"
5018 [(match_operand:VALL_F16 0 "register_operand")
5019 (match_operand:DI 1 "register_operand")]
5020 "TARGET_SIMD"
5021 {
5022 machine_mode mode = <VALL_F16:MODE>mode;
5023 rtx mem = gen_rtx_MEM (mode, operands[1]);
5024
5025 if (BYTES_BIG_ENDIAN)
5026 emit_insn (gen_aarch64_be_ld1<VALL_F16:mode> (operands[0], mem));
5027 else
5028 emit_move_insn (operands[0], mem);
5029 DONE;
5030 })
5031
5032 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
5033 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5034 (match_operand:DI 1 "register_operand" "r")
5035 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5036 "TARGET_SIMD"
5037 {
5038 machine_mode mode = <VSTRUCT:MODE>mode;
5039 rtx mem = gen_rtx_MEM (mode, operands[1]);
5040
5041 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
5042 DONE;
5043 })
5044
5045 (define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5046 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5047 (match_operand:DI 1 "register_operand" "w")
5048 (match_operand:VSTRUCT 2 "register_operand" "0")
5049 (match_operand:SI 3 "immediate_operand" "i")
5050 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5051 "TARGET_SIMD"
5052 {
5053 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5054 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5055 * <VSTRUCT:nregs>);
5056
5057 aarch64_simd_lane_bounds (operands[3], 0,
5058 GET_MODE_NUNITS (<VALLDIF:MODE>mode),
5059 NULL);
5060 emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5061 operands[0], mem, operands[2], operands[3]));
5062 DONE;
5063 })
5064
5065 ;; Expanders for builtins to extract vector registers from large
5066 ;; opaque integer modes.
5067
5068 ;; D-register list.
5069
5070 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
5071 [(match_operand:VDC 0 "register_operand" "=w")
5072 (match_operand:VSTRUCT 1 "register_operand" "w")
5073 (match_operand:SI 2 "immediate_operand" "i")]
5074 "TARGET_SIMD"
5075 {
5076 int part = INTVAL (operands[2]);
5077 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
5078 int offset = part * 16;
5079
5080 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
5081 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
5082 DONE;
5083 })
5084
5085 ;; Q-register list.
5086
5087 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
5088 [(match_operand:VQ 0 "register_operand" "=w")
5089 (match_operand:VSTRUCT 1 "register_operand" "w")
5090 (match_operand:SI 2 "immediate_operand" "i")]
5091 "TARGET_SIMD"
5092 {
5093 int part = INTVAL (operands[2]);
5094 int offset = part * 16;
5095
5096 emit_move_insn (operands[0],
5097 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
5098 DONE;
5099 })
5100
5101 ;; Permuted-store expanders for neon intrinsics.
5102
5103 ;; Permute instructions
5104
5105 ;; vec_perm support
5106
5107 (define_expand "vec_perm_const<mode>"
5108 [(match_operand:VALL_F16 0 "register_operand")
5109 (match_operand:VALL_F16 1 "register_operand")
5110 (match_operand:VALL_F16 2 "register_operand")
5111 (match_operand:<V_cmp_result> 3)]
5112 "TARGET_SIMD"
5113 {
5114 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
5115 operands[2], operands[3]))
5116 DONE;
5117 else
5118 FAIL;
5119 })
5120
5121 (define_expand "vec_perm<mode>"
5122 [(match_operand:VB 0 "register_operand")
5123 (match_operand:VB 1 "register_operand")
5124 (match_operand:VB 2 "register_operand")
5125 (match_operand:VB 3 "register_operand")]
5126 "TARGET_SIMD"
5127 {
5128 aarch64_expand_vec_perm (operands[0], operands[1],
5129 operands[2], operands[3]);
5130 DONE;
5131 })
5132
5133 (define_insn "aarch64_tbl1<mode>"
5134 [(set (match_operand:VB 0 "register_operand" "=w")
5135 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
5136 (match_operand:VB 2 "register_operand" "w")]
5137 UNSPEC_TBL))]
5138 "TARGET_SIMD"
5139 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
5140 [(set_attr "type" "neon_tbl1<q>")]
5141 )
5142
5143 ;; Two source registers.
5144
5145 (define_insn "aarch64_tbl2v16qi"
5146 [(set (match_operand:V16QI 0 "register_operand" "=w")
5147 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
5148 (match_operand:V16QI 2 "register_operand" "w")]
5149 UNSPEC_TBL))]
5150 "TARGET_SIMD"
5151 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
5152 [(set_attr "type" "neon_tbl2_q")]
5153 )
5154
5155 (define_insn "aarch64_tbl3<mode>"
5156 [(set (match_operand:VB 0 "register_operand" "=w")
5157 (unspec:VB [(match_operand:OI 1 "register_operand" "w")
5158 (match_operand:VB 2 "register_operand" "w")]
5159 UNSPEC_TBL))]
5160 "TARGET_SIMD"
5161 "tbl\\t%S0.<Vbtype>, {%S1.16b - %T1.16b}, %S2.<Vbtype>"
5162 [(set_attr "type" "neon_tbl3")]
5163 )
5164
5165 (define_insn "aarch64_tbx4<mode>"
5166 [(set (match_operand:VB 0 "register_operand" "=w")
5167 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5168 (match_operand:OI 2 "register_operand" "w")
5169 (match_operand:VB 3 "register_operand" "w")]
5170 UNSPEC_TBX))]
5171 "TARGET_SIMD"
5172 "tbx\\t%S0.<Vbtype>, {%S2.16b - %T2.16b}, %S3.<Vbtype>"
5173 [(set_attr "type" "neon_tbl4")]
5174 )
5175
5176 ;; Three source registers.
5177
5178 (define_insn "aarch64_qtbl3<mode>"
5179 [(set (match_operand:VB 0 "register_operand" "=w")
5180 (unspec:VB [(match_operand:CI 1 "register_operand" "w")
5181 (match_operand:VB 2 "register_operand" "w")]
5182 UNSPEC_TBL))]
5183 "TARGET_SIMD"
5184 "tbl\\t%S0.<Vbtype>, {%S1.16b - %U1.16b}, %S2.<Vbtype>"
5185 [(set_attr "type" "neon_tbl3")]
5186 )
5187
5188 (define_insn "aarch64_qtbx3<mode>"
5189 [(set (match_operand:VB 0 "register_operand" "=w")
5190 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5191 (match_operand:CI 2 "register_operand" "w")
5192 (match_operand:VB 3 "register_operand" "w")]
5193 UNSPEC_TBX))]
5194 "TARGET_SIMD"
5195 "tbx\\t%S0.<Vbtype>, {%S2.16b - %U2.16b}, %S3.<Vbtype>"
5196 [(set_attr "type" "neon_tbl3")]
5197 )
5198
5199 ;; Four source registers.
5200
5201 (define_insn "aarch64_qtbl4<mode>"
5202 [(set (match_operand:VB 0 "register_operand" "=w")
5203 (unspec:VB [(match_operand:XI 1 "register_operand" "w")
5204 (match_operand:VB 2 "register_operand" "w")]
5205 UNSPEC_TBL))]
5206 "TARGET_SIMD"
5207 "tbl\\t%S0.<Vbtype>, {%S1.16b - %V1.16b}, %S2.<Vbtype>"
5208 [(set_attr "type" "neon_tbl4")]
5209 )
5210
5211 (define_insn "aarch64_qtbx4<mode>"
5212 [(set (match_operand:VB 0 "register_operand" "=w")
5213 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5214 (match_operand:XI 2 "register_operand" "w")
5215 (match_operand:VB 3 "register_operand" "w")]
5216 UNSPEC_TBX))]
5217 "TARGET_SIMD"
5218 "tbx\\t%S0.<Vbtype>, {%S2.16b - %V2.16b}, %S3.<Vbtype>"
5219 [(set_attr "type" "neon_tbl4")]
5220 )
5221
5222 (define_insn_and_split "aarch64_combinev16qi"
5223 [(set (match_operand:OI 0 "register_operand" "=w")
5224 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
5225 (match_operand:V16QI 2 "register_operand" "w")]
5226 UNSPEC_CONCAT))]
5227 "TARGET_SIMD"
5228 "#"
5229 "&& reload_completed"
5230 [(const_int 0)]
5231 {
5232 aarch64_split_combinev16qi (operands);
5233 DONE;
5234 }
5235 [(set_attr "type" "multiple")]
5236 )
5237
5238 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
5239 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5240 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5241 (match_operand:VALL_F16 2 "register_operand" "w")]
5242 PERMUTE))]
5243 "TARGET_SIMD"
5244 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
5245 [(set_attr "type" "neon_permute<q>")]
5246 )
5247
5248 ;; Note immediate (third) operand is lane index not byte index.
5249 (define_insn "aarch64_ext<mode>"
5250 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5251 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5252 (match_operand:VALL_F16 2 "register_operand" "w")
5253 (match_operand:SI 3 "immediate_operand" "i")]
5254 UNSPEC_EXT))]
5255 "TARGET_SIMD"
5256 {
5257 operands[3] = GEN_INT (INTVAL (operands[3])
5258 * GET_MODE_UNIT_SIZE (<MODE>mode));
5259 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
5260 }
5261 [(set_attr "type" "neon_ext<q>")]
5262 )
5263
5264 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
5265 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5266 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")]
5267 REVERSE))]
5268 "TARGET_SIMD"
5269 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
5270 [(set_attr "type" "neon_rev<q>")]
5271 )
5272
5273 (define_insn "aarch64_st2<mode>_dreg"
5274 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5275 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5276 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5277 UNSPEC_ST2))]
5278 "TARGET_SIMD"
5279 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
5280 [(set_attr "type" "neon_store2_2reg")]
5281 )
5282
5283 (define_insn "aarch64_st2<mode>_dreg"
5284 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5285 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5286 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5287 UNSPEC_ST2))]
5288 "TARGET_SIMD"
5289 "st1\\t{%S1.1d - %T1.1d}, %0"
5290 [(set_attr "type" "neon_store1_2reg")]
5291 )
5292
5293 (define_insn "aarch64_st3<mode>_dreg"
5294 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5295 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5296 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5297 UNSPEC_ST3))]
5298 "TARGET_SIMD"
5299 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
5300 [(set_attr "type" "neon_store3_3reg")]
5301 )
5302
5303 (define_insn "aarch64_st3<mode>_dreg"
5304 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5305 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5306 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5307 UNSPEC_ST3))]
5308 "TARGET_SIMD"
5309 "st1\\t{%S1.1d - %U1.1d}, %0"
5310 [(set_attr "type" "neon_store1_3reg")]
5311 )
5312
5313 (define_insn "aarch64_st4<mode>_dreg"
5314 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5315 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5316 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5317 UNSPEC_ST4))]
5318 "TARGET_SIMD"
5319 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
5320 [(set_attr "type" "neon_store4_4reg")]
5321 )
5322
5323 (define_insn "aarch64_st4<mode>_dreg"
5324 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5325 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5326 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5327 UNSPEC_ST4))]
5328 "TARGET_SIMD"
5329 "st1\\t{%S1.1d - %V1.1d}, %0"
5330 [(set_attr "type" "neon_store1_4reg")]
5331 )
5332
5333 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
5334 [(match_operand:DI 0 "register_operand" "r")
5335 (match_operand:VSTRUCT 1 "register_operand" "w")
5336 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5337 "TARGET_SIMD"
5338 {
5339 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5340 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5341
5342 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
5343 DONE;
5344 })
5345
5346 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
5347 [(match_operand:DI 0 "register_operand" "r")
5348 (match_operand:VSTRUCT 1 "register_operand" "w")
5349 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5350 "TARGET_SIMD"
5351 {
5352 machine_mode mode = <VSTRUCT:MODE>mode;
5353 rtx mem = gen_rtx_MEM (mode, operands[0]);
5354
5355 emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
5356 DONE;
5357 })
5358
5359 (define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5360 [(match_operand:DI 0 "register_operand" "r")
5361 (match_operand:VSTRUCT 1 "register_operand" "w")
5362 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
5363 (match_operand:SI 2 "immediate_operand")]
5364 "TARGET_SIMD"
5365 {
5366 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5367 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5368 * <VSTRUCT:nregs>);
5369
5370 emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5371 mem, operands[1], operands[2]));
5372 DONE;
5373 })
5374
5375 (define_expand "aarch64_st1<VALL_F16:mode>"
5376 [(match_operand:DI 0 "register_operand")
5377 (match_operand:VALL_F16 1 "register_operand")]
5378 "TARGET_SIMD"
5379 {
5380 machine_mode mode = <VALL_F16:MODE>mode;
5381 rtx mem = gen_rtx_MEM (mode, operands[0]);
5382
5383 if (BYTES_BIG_ENDIAN)
5384 emit_insn (gen_aarch64_be_st1<VALL_F16:mode> (mem, operands[1]));
5385 else
5386 emit_move_insn (mem, operands[1]);
5387 DONE;
5388 })
5389
5390 ;; Expander for builtins to insert vector registers into large
5391 ;; opaque integer modes.
5392
5393 ;; Q-register list. We don't need a D-reg inserter as we zero
5394 ;; extend them in arm_neon.h and insert the resulting Q-regs.
5395
5396 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
5397 [(match_operand:VSTRUCT 0 "register_operand" "+w")
5398 (match_operand:VSTRUCT 1 "register_operand" "0")
5399 (match_operand:VQ 2 "register_operand" "w")
5400 (match_operand:SI 3 "immediate_operand" "i")]
5401 "TARGET_SIMD"
5402 {
5403 int part = INTVAL (operands[3]);
5404 int offset = part * 16;
5405
5406 emit_move_insn (operands[0], operands[1]);
5407 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
5408 operands[2]);
5409 DONE;
5410 })
5411
5412 ;; Standard pattern name vec_init<mode>.
5413
5414 (define_expand "vec_init<mode>"
5415 [(match_operand:VALL_F16 0 "register_operand" "")
5416 (match_operand 1 "" "")]
5417 "TARGET_SIMD"
5418 {
5419 aarch64_expand_vector_init (operands[0], operands[1]);
5420 DONE;
5421 })
5422
5423 (define_insn "*aarch64_simd_ld1r<mode>"
5424 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5425 (vec_duplicate:VALL_F16
5426 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
5427 "TARGET_SIMD"
5428 "ld1r\\t{%0.<Vtype>}, %1"
5429 [(set_attr "type" "neon_load1_all_lanes")]
5430 )
5431
5432 (define_insn "aarch64_frecpe<mode>"
5433 [(set (match_operand:VHSDF 0 "register_operand" "=w")
5434 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
5435 UNSPEC_FRECPE))]
5436 "TARGET_SIMD"
5437 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
5438 [(set_attr "type" "neon_fp_recpe_<stype><q>")]
5439 )
5440
5441 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
5442 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5443 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5444 FRECP))]
5445 "TARGET_SIMD"
5446 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
5447 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF_F16:stype>")]
5448 )
5449
5450 (define_insn "aarch64_frecps<mode>"
5451 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
5452 (unspec:VHSDF_HSDF
5453 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
5454 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
5455 UNSPEC_FRECPS))]
5456 "TARGET_SIMD"
5457 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
5458 [(set_attr "type" "neon_fp_recps_<stype><q>")]
5459 )
5460
5461 (define_insn "aarch64_urecpe<mode>"
5462 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
5463 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
5464 UNSPEC_URECPE))]
5465 "TARGET_SIMD"
5466 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
5467 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
5468
5469 ;; Standard pattern name vec_extract<mode>.
5470
5471 (define_expand "vec_extract<mode>"
5472 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
5473 (match_operand:VALL_F16 1 "register_operand" "")
5474 (match_operand:SI 2 "immediate_operand" "")]
5475 "TARGET_SIMD"
5476 {
5477 emit_insn
5478 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
5479 DONE;
5480 })
5481
5482 ;; aes
5483
5484 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
5485 [(set (match_operand:V16QI 0 "register_operand" "=w")
5486 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
5487 (match_operand:V16QI 2 "register_operand" "w")]
5488 CRYPTO_AES))]
5489 "TARGET_SIMD && TARGET_CRYPTO"
5490 "aes<aes_op>\\t%0.16b, %2.16b"
5491 [(set_attr "type" "crypto_aese")]
5492 )
5493
5494 ;; When AES/AESMC fusion is enabled we want the register allocation to
5495 ;; look like:
5496 ;; AESE Vn, _
5497 ;; AESMC Vn, Vn
5498 ;; So prefer to tie operand 1 to operand 0 when fusing.
5499
5500 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
5501 [(set (match_operand:V16QI 0 "register_operand" "=w,w")
5502 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0,w")]
5503 CRYPTO_AESMC))]
5504 "TARGET_SIMD && TARGET_CRYPTO"
5505 "aes<aesmc_op>\\t%0.16b, %1.16b"
5506 [(set_attr "type" "crypto_aesmc")
5507 (set_attr_alternative "enabled"
5508 [(if_then_else (match_test
5509 "aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)")
5510 (const_string "yes" )
5511 (const_string "no"))
5512 (const_string "yes")])]
5513 )
5514
5515 ;; sha1
5516
5517 (define_insn "aarch64_crypto_sha1hsi"
5518 [(set (match_operand:SI 0 "register_operand" "=w")
5519 (unspec:SI [(match_operand:SI 1
5520 "register_operand" "w")]
5521 UNSPEC_SHA1H))]
5522 "TARGET_SIMD && TARGET_CRYPTO"
5523 "sha1h\\t%s0, %s1"
5524 [(set_attr "type" "crypto_sha1_fast")]
5525 )
5526
5527 (define_insn "aarch64_crypto_sha1su1v4si"
5528 [(set (match_operand:V4SI 0 "register_operand" "=w")
5529 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5530 (match_operand:V4SI 2 "register_operand" "w")]
5531 UNSPEC_SHA1SU1))]
5532 "TARGET_SIMD && TARGET_CRYPTO"
5533 "sha1su1\\t%0.4s, %2.4s"
5534 [(set_attr "type" "crypto_sha1_fast")]
5535 )
5536
5537 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5538 [(set (match_operand:V4SI 0 "register_operand" "=w")
5539 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5540 (match_operand:SI 2 "register_operand" "w")
5541 (match_operand:V4SI 3 "register_operand" "w")]
5542 CRYPTO_SHA1))]
5543 "TARGET_SIMD && TARGET_CRYPTO"
5544 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5545 [(set_attr "type" "crypto_sha1_slow")]
5546 )
5547
5548 (define_insn "aarch64_crypto_sha1su0v4si"
5549 [(set (match_operand:V4SI 0 "register_operand" "=w")
5550 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5551 (match_operand:V4SI 2 "register_operand" "w")
5552 (match_operand:V4SI 3 "register_operand" "w")]
5553 UNSPEC_SHA1SU0))]
5554 "TARGET_SIMD && TARGET_CRYPTO"
5555 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5556 [(set_attr "type" "crypto_sha1_xor")]
5557 )
5558
5559 ;; sha256
5560
5561 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5562 [(set (match_operand:V4SI 0 "register_operand" "=w")
5563 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5564 (match_operand:V4SI 2 "register_operand" "w")
5565 (match_operand:V4SI 3 "register_operand" "w")]
5566 CRYPTO_SHA256))]
5567 "TARGET_SIMD && TARGET_CRYPTO"
5568 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5569 [(set_attr "type" "crypto_sha256_slow")]
5570 )
5571
5572 (define_insn "aarch64_crypto_sha256su0v4si"
5573 [(set (match_operand:V4SI 0 "register_operand" "=w")
5574 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5575 (match_operand:V4SI 2 "register_operand" "w")]
5576 UNSPEC_SHA256SU0))]
5577 "TARGET_SIMD &&TARGET_CRYPTO"
5578 "sha256su0\\t%0.4s, %2.4s"
5579 [(set_attr "type" "crypto_sha256_fast")]
5580 )
5581
5582 (define_insn "aarch64_crypto_sha256su1v4si"
5583 [(set (match_operand:V4SI 0 "register_operand" "=w")
5584 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5585 (match_operand:V4SI 2 "register_operand" "w")
5586 (match_operand:V4SI 3 "register_operand" "w")]
5587 UNSPEC_SHA256SU1))]
5588 "TARGET_SIMD &&TARGET_CRYPTO"
5589 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5590 [(set_attr "type" "crypto_sha256_slow")]
5591 )
5592
5593 ;; pmull
5594
5595 (define_insn "aarch64_crypto_pmulldi"
5596 [(set (match_operand:TI 0 "register_operand" "=w")
5597 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5598 (match_operand:DI 2 "register_operand" "w")]
5599 UNSPEC_PMULL))]
5600 "TARGET_SIMD && TARGET_CRYPTO"
5601 "pmull\\t%0.1q, %1.1d, %2.1d"
5602 [(set_attr "type" "neon_mul_d_long")]
5603 )
5604
5605 (define_insn "aarch64_crypto_pmullv2di"
5606 [(set (match_operand:TI 0 "register_operand" "=w")
5607 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5608 (match_operand:V2DI 2 "register_operand" "w")]
5609 UNSPEC_PMULL2))]
5610 "TARGET_SIMD && TARGET_CRYPTO"
5611 "pmull2\\t%0.1q, %1.2d, %2.2d"
5612 [(set_attr "type" "neon_mul_d_long")]
5613 )
Mirror of https://gcc.gnu.org/git/gcc.git