]> git.ipfire.org Git - thirdparty/gcc.git/blob - gcc/config/aarch64/aarch64-simd.md
projects / thirdparty / gcc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
gcc/
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64-simd.md
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2015 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
20
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
23 (match_operand:VALL 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
30
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
43
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ_I
47 (match_operand:<VEL> 1 "register_operand" "r, w")))]
48 "TARGET_SIMD"
49 "@
50 dup\\t%0.<Vtype>, %<vw>1
51 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
52 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
53 )
54
55 (define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF 0 "register_operand" "=w")
57 (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
58 "TARGET_SIMD"
59 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
60 [(set_attr "type" "neon_dup<q>")]
61 )
62
63 (define_insn "aarch64_dup_lane<mode>"
64 [(set (match_operand:VALL 0 "register_operand" "=w")
65 (vec_duplicate:VALL
66 (vec_select:<VEL>
67 (match_operand:VALL 1 "register_operand" "w")
68 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
69 )))]
70 "TARGET_SIMD"
71 {
72 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
73 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
74 }
75 [(set_attr "type" "neon_dup<q>")]
76 )
77
78 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
79 [(set (match_operand:VALL 0 "register_operand" "=w")
80 (vec_duplicate:VALL
81 (vec_select:<VEL>
82 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
83 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
84 )))]
85 "TARGET_SIMD"
86 {
87 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
88 INTVAL (operands[2])));
89 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
90 }
91 [(set_attr "type" "neon_dup<q>")]
92 )
93
94 (define_insn "*aarch64_simd_mov<mode>"
95 [(set (match_operand:VD 0 "nonimmediate_operand"
96 "=w, m, w, ?r, ?w, ?r, w")
97 (match_operand:VD 1 "general_operand"
98 "m, w, w, w, r, r, Dn"))]
99 "TARGET_SIMD
100 && (register_operand (operands[0], <MODE>mode)
101 || register_operand (operands[1], <MODE>mode))"
102 {
103 switch (which_alternative)
104 {
105 case 0: return "ldr\\t%d0, %1";
106 case 1: return "str\\t%d1, %0";
107 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
108 case 3: return "umov\t%0, %1.d[0]";
109 case 4: return "ins\t%0.d[0], %1";
110 case 5: return "mov\t%0, %1";
111 case 6:
112 return aarch64_output_simd_mov_immediate (operands[1],
113 <MODE>mode, 64);
114 default: gcc_unreachable ();
115 }
116 }
117 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
118 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
119 mov_reg, neon_move<q>")]
120 )
121
122 (define_insn "*aarch64_simd_mov<mode>"
123 [(set (match_operand:VQ 0 "nonimmediate_operand"
124 "=w, m, w, ?r, ?w, ?r, w")
125 (match_operand:VQ 1 "general_operand"
126 "m, w, w, w, r, r, Dn"))]
127 "TARGET_SIMD
128 && (register_operand (operands[0], <MODE>mode)
129 || register_operand (operands[1], <MODE>mode))"
130 {
131 switch (which_alternative)
132 {
133 case 0:
134 return "ldr\\t%q0, %1";
135 case 1:
136 return "str\\t%q1, %0";
137 case 2:
138 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
139 case 3:
140 case 4:
141 case 5:
142 return "#";
143 case 6:
144 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
145 default:
146 gcc_unreachable ();
147 }
148 }
149 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
150 neon_logic<q>, multiple, multiple, multiple,\
151 neon_move<q>")
152 (set_attr "length" "4,4,4,8,8,8,4")]
153 )
154
155 (define_split
156 [(set (match_operand:VQ 0 "register_operand" "")
157 (match_operand:VQ 1 "register_operand" ""))]
158 "TARGET_SIMD && reload_completed
159 && GP_REGNUM_P (REGNO (operands[0]))
160 && GP_REGNUM_P (REGNO (operands[1]))"
161 [(const_int 0)]
162 {
163 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
164 DONE;
165 })
166
167 (define_split
168 [(set (match_operand:VQ 0 "register_operand" "")
169 (match_operand:VQ 1 "register_operand" ""))]
170 "TARGET_SIMD && reload_completed
171 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
172 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
173 [(const_int 0)]
174 {
175 aarch64_split_simd_move (operands[0], operands[1]);
176 DONE;
177 })
178
179 (define_expand "aarch64_split_simd_mov<mode>"
180 [(set (match_operand:VQ 0)
181 (match_operand:VQ 1))]
182 "TARGET_SIMD"
183 {
184 rtx dst = operands[0];
185 rtx src = operands[1];
186
187 if (GP_REGNUM_P (REGNO (src)))
188 {
189 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
190 rtx src_high_part = gen_highpart (<VHALF>mode, src);
191
192 emit_insn
193 (gen_move_lo_quad_<mode> (dst, src_low_part));
194 emit_insn
195 (gen_move_hi_quad_<mode> (dst, src_high_part));
196 }
197
198 else
199 {
200 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
201 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
202 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
203 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
204
205 emit_insn
206 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
207 emit_insn
208 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
209 }
210 DONE;
211 }
212 )
213
214 (define_insn "aarch64_simd_mov_from_<mode>low"
215 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
216 (vec_select:<VHALF>
217 (match_operand:VQ 1 "register_operand" "w")
218 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
219 "TARGET_SIMD && reload_completed"
220 "umov\t%0, %1.d[0]"
221 [(set_attr "type" "neon_to_gp<q>")
222 (set_attr "length" "4")
223 ])
224
225 (define_insn "aarch64_simd_mov_from_<mode>high"
226 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
227 (vec_select:<VHALF>
228 (match_operand:VQ 1 "register_operand" "w")
229 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
230 "TARGET_SIMD && reload_completed"
231 "umov\t%0, %1.d[1]"
232 [(set_attr "type" "neon_to_gp<q>")
233 (set_attr "length" "4")
234 ])
235
236 (define_insn "orn<mode>3"
237 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
238 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
239 (match_operand:VDQ_I 2 "register_operand" "w")))]
240 "TARGET_SIMD"
241 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
242 [(set_attr "type" "neon_logic<q>")]
243 )
244
245 (define_insn "bic<mode>3"
246 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
247 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
248 (match_operand:VDQ_I 2 "register_operand" "w")))]
249 "TARGET_SIMD"
250 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
251 [(set_attr "type" "neon_logic<q>")]
252 )
253
254 (define_insn "add<mode>3"
255 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
256 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
257 (match_operand:VDQ_I 2 "register_operand" "w")))]
258 "TARGET_SIMD"
259 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
260 [(set_attr "type" "neon_add<q>")]
261 )
262
263 (define_insn "sub<mode>3"
264 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
265 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
266 (match_operand:VDQ_I 2 "register_operand" "w")))]
267 "TARGET_SIMD"
268 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
269 [(set_attr "type" "neon_sub<q>")]
270 )
271
272 (define_insn "mul<mode>3"
273 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
274 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
275 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
276 "TARGET_SIMD"
277 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
278 [(set_attr "type" "neon_mul_<Vetype><q>")]
279 )
280
281 (define_insn "bswap<mode>2"
282 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
283 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
284 "TARGET_SIMD"
285 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
286 [(set_attr "type" "neon_rev<q>")]
287 )
288
289 (define_insn "aarch64_rbit<mode>"
290 [(set (match_operand:VB 0 "register_operand" "=w")
291 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
292 UNSPEC_RBIT))]
293 "TARGET_SIMD"
294 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
295 [(set_attr "type" "neon_rbit")]
296 )
297
298 (define_expand "ctz<mode>2"
299 [(set (match_operand:VS 0 "register_operand")
300 (ctz:VS (match_operand:VS 1 "register_operand")))]
301 "TARGET_SIMD"
302 {
303 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
304 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
305 <MODE>mode, 0);
306 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
307 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
308 DONE;
309 }
310 )
311
312 (define_insn "*aarch64_mul3_elt<mode>"
313 [(set (match_operand:VMUL 0 "register_operand" "=w")
314 (mult:VMUL
315 (vec_duplicate:VMUL
316 (vec_select:<VEL>
317 (match_operand:VMUL 1 "register_operand" "<h_con>")
318 (parallel [(match_operand:SI 2 "immediate_operand")])))
319 (match_operand:VMUL 3 "register_operand" "w")))]
320 "TARGET_SIMD"
321 {
322 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
323 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
324 }
325 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
326 )
327
328 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
329 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
330 (mult:VMUL_CHANGE_NLANES
331 (vec_duplicate:VMUL_CHANGE_NLANES
332 (vec_select:<VEL>
333 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
334 (parallel [(match_operand:SI 2 "immediate_operand")])))
335 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
336 "TARGET_SIMD"
337 {
338 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
339 INTVAL (operands[2])));
340 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
341 }
342 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
343 )
344
345 (define_insn "*aarch64_mul3_elt_to_128df"
346 [(set (match_operand:V2DF 0 "register_operand" "=w")
347 (mult:V2DF
348 (vec_duplicate:V2DF
349 (match_operand:DF 2 "register_operand" "w"))
350 (match_operand:V2DF 1 "register_operand" "w")))]
351 "TARGET_SIMD"
352 "fmul\\t%0.2d, %1.2d, %2.d[0]"
353 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
354 )
355
356 (define_insn "*aarch64_mul3_elt_to_64v2df"
357 [(set (match_operand:DF 0 "register_operand" "=w")
358 (mult:DF
359 (vec_select:DF
360 (match_operand:V2DF 1 "register_operand" "w")
361 (parallel [(match_operand:SI 2 "immediate_operand")]))
362 (match_operand:DF 3 "register_operand" "w")))]
363 "TARGET_SIMD"
364 {
365 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
366 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
367 }
368 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
369 )
370
371 (define_insn "neg<mode>2"
372 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
373 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
374 "TARGET_SIMD"
375 "neg\t%0.<Vtype>, %1.<Vtype>"
376 [(set_attr "type" "neon_neg<q>")]
377 )
378
379 (define_insn "abs<mode>2"
380 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
381 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
382 "TARGET_SIMD"
383 "abs\t%0.<Vtype>, %1.<Vtype>"
384 [(set_attr "type" "neon_abs<q>")]
385 )
386
387 (define_insn "abd<mode>_3"
388 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
389 (abs:VDQ_BHSI (minus:VDQ_BHSI
390 (match_operand:VDQ_BHSI 1 "register_operand" "w")
391 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
392 "TARGET_SIMD"
393 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
394 [(set_attr "type" "neon_abd<q>")]
395 )
396
397 (define_insn "aba<mode>_3"
398 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
399 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
400 (match_operand:VDQ_BHSI 1 "register_operand" "w")
401 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
402 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
403 "TARGET_SIMD"
404 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
405 [(set_attr "type" "neon_arith_acc<q>")]
406 )
407
408 (define_insn "fabd<mode>_3"
409 [(set (match_operand:VDQF 0 "register_operand" "=w")
410 (abs:VDQF (minus:VDQF
411 (match_operand:VDQF 1 "register_operand" "w")
412 (match_operand:VDQF 2 "register_operand" "w"))))]
413 "TARGET_SIMD"
414 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
415 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
416 )
417
418 (define_insn "*fabd_scalar<mode>3"
419 [(set (match_operand:GPF 0 "register_operand" "=w")
420 (abs:GPF (minus:GPF
421 (match_operand:GPF 1 "register_operand" "w")
422 (match_operand:GPF 2 "register_operand" "w"))))]
423 "TARGET_SIMD"
424 "fabd\t%<s>0, %<s>1, %<s>2"
425 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
426 )
427
428 (define_insn "and<mode>3"
429 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
430 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
431 (match_operand:VDQ_I 2 "register_operand" "w")))]
432 "TARGET_SIMD"
433 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
434 [(set_attr "type" "neon_logic<q>")]
435 )
436
437 (define_insn "ior<mode>3"
438 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
439 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
440 (match_operand:VDQ_I 2 "register_operand" "w")))]
441 "TARGET_SIMD"
442 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
443 [(set_attr "type" "neon_logic<q>")]
444 )
445
446 (define_insn "xor<mode>3"
447 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
448 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
449 (match_operand:VDQ_I 2 "register_operand" "w")))]
450 "TARGET_SIMD"
451 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
452 [(set_attr "type" "neon_logic<q>")]
453 )
454
455 (define_insn "one_cmpl<mode>2"
456 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
457 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
458 "TARGET_SIMD"
459 "not\t%0.<Vbtype>, %1.<Vbtype>"
460 [(set_attr "type" "neon_logic<q>")]
461 )
462
463 (define_insn "aarch64_simd_vec_set<mode>"
464 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
465 (vec_merge:VDQ_BHSI
466 (vec_duplicate:VDQ_BHSI
467 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
468 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
469 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
470 "TARGET_SIMD"
471 {
472 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
473 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
474 switch (which_alternative)
475 {
476 case 0:
477 return "ins\\t%0.<Vetype>[%p2], %w1";
478 case 1:
479 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
480 case 2:
481 return "ld1\\t{%0.<Vetype>}[%p2], %1";
482 default:
483 gcc_unreachable ();
484 }
485 }
486 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_1reg<q>")]
487 )
488
489 (define_insn "aarch64_simd_lshr<mode>"
490 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
491 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
492 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
493 "TARGET_SIMD"
494 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
495 [(set_attr "type" "neon_shift_imm<q>")]
496 )
497
498 (define_insn "aarch64_simd_ashr<mode>"
499 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
500 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
501 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
502 "TARGET_SIMD"
503 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
504 [(set_attr "type" "neon_shift_imm<q>")]
505 )
506
507 (define_insn "aarch64_simd_imm_shl<mode>"
508 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
509 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
510 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
511 "TARGET_SIMD"
512 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
513 [(set_attr "type" "neon_shift_imm<q>")]
514 )
515
516 (define_insn "aarch64_simd_reg_sshl<mode>"
517 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
518 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
519 (match_operand:VDQ_I 2 "register_operand" "w")))]
520 "TARGET_SIMD"
521 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
522 [(set_attr "type" "neon_shift_reg<q>")]
523 )
524
525 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
526 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
527 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
528 (match_operand:VDQ_I 2 "register_operand" "w")]
529 UNSPEC_ASHIFT_UNSIGNED))]
530 "TARGET_SIMD"
531 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
532 [(set_attr "type" "neon_shift_reg<q>")]
533 )
534
535 (define_insn "aarch64_simd_reg_shl<mode>_signed"
536 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
537 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
538 (match_operand:VDQ_I 2 "register_operand" "w")]
539 UNSPEC_ASHIFT_SIGNED))]
540 "TARGET_SIMD"
541 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
542 [(set_attr "type" "neon_shift_reg<q>")]
543 )
544
545 (define_expand "ashl<mode>3"
546 [(match_operand:VDQ_I 0 "register_operand" "")
547 (match_operand:VDQ_I 1 "register_operand" "")
548 (match_operand:SI 2 "general_operand" "")]
549 "TARGET_SIMD"
550 {
551 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
552 int shift_amount;
553
554 if (CONST_INT_P (operands[2]))
555 {
556 shift_amount = INTVAL (operands[2]);
557 if (shift_amount >= 0 && shift_amount < bit_width)
558 {
559 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
560 shift_amount);
561 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
562 operands[1],
563 tmp));
564 DONE;
565 }
566 else
567 {
568 operands[2] = force_reg (SImode, operands[2]);
569 }
570 }
571 else if (MEM_P (operands[2]))
572 {
573 operands[2] = force_reg (SImode, operands[2]);
574 }
575
576 if (REG_P (operands[2]))
577 {
578 rtx tmp = gen_reg_rtx (<MODE>mode);
579 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
580 convert_to_mode (<VEL>mode,
581 operands[2],
582 0)));
583 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
584 tmp));
585 DONE;
586 }
587 else
588 FAIL;
589 }
590 )
591
592 (define_expand "lshr<mode>3"
593 [(match_operand:VDQ_I 0 "register_operand" "")
594 (match_operand:VDQ_I 1 "register_operand" "")
595 (match_operand:SI 2 "general_operand" "")]
596 "TARGET_SIMD"
597 {
598 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
599 int shift_amount;
600
601 if (CONST_INT_P (operands[2]))
602 {
603 shift_amount = INTVAL (operands[2]);
604 if (shift_amount > 0 && shift_amount <= bit_width)
605 {
606 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
607 shift_amount);
608 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
609 operands[1],
610 tmp));
611 DONE;
612 }
613 else
614 operands[2] = force_reg (SImode, operands[2]);
615 }
616 else if (MEM_P (operands[2]))
617 {
618 operands[2] = force_reg (SImode, operands[2]);
619 }
620
621 if (REG_P (operands[2]))
622 {
623 rtx tmp = gen_reg_rtx (SImode);
624 rtx tmp1 = gen_reg_rtx (<MODE>mode);
625 emit_insn (gen_negsi2 (tmp, operands[2]));
626 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
627 convert_to_mode (<VEL>mode,
628 tmp, 0)));
629 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
630 operands[1],
631 tmp1));
632 DONE;
633 }
634 else
635 FAIL;
636 }
637 )
638
639 (define_expand "ashr<mode>3"
640 [(match_operand:VDQ_I 0 "register_operand" "")
641 (match_operand:VDQ_I 1 "register_operand" "")
642 (match_operand:SI 2 "general_operand" "")]
643 "TARGET_SIMD"
644 {
645 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
646 int shift_amount;
647
648 if (CONST_INT_P (operands[2]))
649 {
650 shift_amount = INTVAL (operands[2]);
651 if (shift_amount > 0 && shift_amount <= bit_width)
652 {
653 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
654 shift_amount);
655 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
656 operands[1],
657 tmp));
658 DONE;
659 }
660 else
661 operands[2] = force_reg (SImode, operands[2]);
662 }
663 else if (MEM_P (operands[2]))
664 {
665 operands[2] = force_reg (SImode, operands[2]);
666 }
667
668 if (REG_P (operands[2]))
669 {
670 rtx tmp = gen_reg_rtx (SImode);
671 rtx tmp1 = gen_reg_rtx (<MODE>mode);
672 emit_insn (gen_negsi2 (tmp, operands[2]));
673 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
674 convert_to_mode (<VEL>mode,
675 tmp, 0)));
676 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
677 operands[1],
678 tmp1));
679 DONE;
680 }
681 else
682 FAIL;
683 }
684 )
685
686 (define_expand "vashl<mode>3"
687 [(match_operand:VDQ_I 0 "register_operand" "")
688 (match_operand:VDQ_I 1 "register_operand" "")
689 (match_operand:VDQ_I 2 "register_operand" "")]
690 "TARGET_SIMD"
691 {
692 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
693 operands[2]));
694 DONE;
695 })
696
697 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
698 ;; Negating individual lanes most certainly offsets the
699 ;; gain from vectorization.
700 (define_expand "vashr<mode>3"
701 [(match_operand:VDQ_BHSI 0 "register_operand" "")
702 (match_operand:VDQ_BHSI 1 "register_operand" "")
703 (match_operand:VDQ_BHSI 2 "register_operand" "")]
704 "TARGET_SIMD"
705 {
706 rtx neg = gen_reg_rtx (<MODE>mode);
707 emit (gen_neg<mode>2 (neg, operands[2]));
708 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
709 neg));
710 DONE;
711 })
712
713 ;; DI vector shift
714 (define_expand "aarch64_ashr_simddi"
715 [(match_operand:DI 0 "register_operand" "=w")
716 (match_operand:DI 1 "register_operand" "w")
717 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
718 "TARGET_SIMD"
719 {
720 /* An arithmetic shift right by 64 fills the result with copies of the sign
721 bit, just like asr by 63 - however the standard pattern does not handle
722 a shift by 64. */
723 if (INTVAL (operands[2]) == 64)
724 operands[2] = GEN_INT (63);
725 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
726 DONE;
727 }
728 )
729
730 (define_expand "vlshr<mode>3"
731 [(match_operand:VDQ_BHSI 0 "register_operand" "")
732 (match_operand:VDQ_BHSI 1 "register_operand" "")
733 (match_operand:VDQ_BHSI 2 "register_operand" "")]
734 "TARGET_SIMD"
735 {
736 rtx neg = gen_reg_rtx (<MODE>mode);
737 emit (gen_neg<mode>2 (neg, operands[2]));
738 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
739 neg));
740 DONE;
741 })
742
743 (define_expand "aarch64_lshr_simddi"
744 [(match_operand:DI 0 "register_operand" "=w")
745 (match_operand:DI 1 "register_operand" "w")
746 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
747 "TARGET_SIMD"
748 {
749 if (INTVAL (operands[2]) == 64)
750 emit_move_insn (operands[0], const0_rtx);
751 else
752 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
753 DONE;
754 }
755 )
756
757 (define_expand "vec_set<mode>"
758 [(match_operand:VDQ_BHSI 0 "register_operand")
759 (match_operand:<VEL> 1 "register_operand")
760 (match_operand:SI 2 "immediate_operand")]
761 "TARGET_SIMD"
762 {
763 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
764 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
765 GEN_INT (elem), operands[0]));
766 DONE;
767 }
768 )
769
770 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
771 (define_insn "vec_shr_<mode>"
772 [(set (match_operand:VD 0 "register_operand" "=w")
773 (lshiftrt:VD (match_operand:VD 1 "register_operand" "w")
774 (match_operand:SI 2 "immediate_operand" "i")))]
775 "TARGET_SIMD"
776 {
777 if (BYTES_BIG_ENDIAN)
778 return "ushl %d0, %d1, %2";
779 else
780 return "ushr %d0, %d1, %2";
781 }
782 [(set_attr "type" "neon_shift_imm")]
783 )
784
785 (define_insn "aarch64_simd_vec_setv2di"
786 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
787 (vec_merge:V2DI
788 (vec_duplicate:V2DI
789 (match_operand:DI 1 "register_operand" "r,w"))
790 (match_operand:V2DI 3 "register_operand" "0,0")
791 (match_operand:SI 2 "immediate_operand" "i,i")))]
792 "TARGET_SIMD"
793 {
794 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
795 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
796 switch (which_alternative)
797 {
798 case 0:
799 return "ins\\t%0.d[%p2], %1";
800 case 1:
801 return "ins\\t%0.d[%p2], %1.d[0]";
802 default:
803 gcc_unreachable ();
804 }
805 }
806 [(set_attr "type" "neon_from_gp, neon_ins_q")]
807 )
808
809 (define_expand "vec_setv2di"
810 [(match_operand:V2DI 0 "register_operand")
811 (match_operand:DI 1 "register_operand")
812 (match_operand:SI 2 "immediate_operand")]
813 "TARGET_SIMD"
814 {
815 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
816 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
817 GEN_INT (elem), operands[0]));
818 DONE;
819 }
820 )
821
822 (define_insn "aarch64_simd_vec_set<mode>"
823 [(set (match_operand:VDQF 0 "register_operand" "=w")
824 (vec_merge:VDQF
825 (vec_duplicate:VDQF
826 (match_operand:<VEL> 1 "register_operand" "w"))
827 (match_operand:VDQF 3 "register_operand" "0")
828 (match_operand:SI 2 "immediate_operand" "i")))]
829 "TARGET_SIMD"
830 {
831 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
832
833 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
834 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
835 }
836 [(set_attr "type" "neon_ins<q>")]
837 )
838
839 (define_expand "vec_set<mode>"
840 [(match_operand:VDQF 0 "register_operand" "+w")
841 (match_operand:<VEL> 1 "register_operand" "w")
842 (match_operand:SI 2 "immediate_operand" "")]
843 "TARGET_SIMD"
844 {
845 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
846 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
847 GEN_INT (elem), operands[0]));
848 DONE;
849 }
850 )
851
852
853 (define_insn "aarch64_mla<mode>"
854 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
855 (plus:VDQ_BHSI (mult:VDQ_BHSI
856 (match_operand:VDQ_BHSI 2 "register_operand" "w")
857 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
858 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
859 "TARGET_SIMD"
860 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
861 [(set_attr "type" "neon_mla_<Vetype><q>")]
862 )
863
864 (define_insn "*aarch64_mla_elt<mode>"
865 [(set (match_operand:VDQHS 0 "register_operand" "=w")
866 (plus:VDQHS
867 (mult:VDQHS
868 (vec_duplicate:VDQHS
869 (vec_select:<VEL>
870 (match_operand:VDQHS 1 "register_operand" "<h_con>")
871 (parallel [(match_operand:SI 2 "immediate_operand")])))
872 (match_operand:VDQHS 3 "register_operand" "w"))
873 (match_operand:VDQHS 4 "register_operand" "0")))]
874 "TARGET_SIMD"
875 {
876 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
877 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
878 }
879 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
880 )
881
882 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
883 [(set (match_operand:VDQHS 0 "register_operand" "=w")
884 (plus:VDQHS
885 (mult:VDQHS
886 (vec_duplicate:VDQHS
887 (vec_select:<VEL>
888 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
889 (parallel [(match_operand:SI 2 "immediate_operand")])))
890 (match_operand:VDQHS 3 "register_operand" "w"))
891 (match_operand:VDQHS 4 "register_operand" "0")))]
892 "TARGET_SIMD"
893 {
894 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
895 INTVAL (operands[2])));
896 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
897 }
898 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
899 )
900
901 (define_insn "aarch64_mls<mode>"
902 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
903 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
904 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
905 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
906 "TARGET_SIMD"
907 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
908 [(set_attr "type" "neon_mla_<Vetype><q>")]
909 )
910
911 (define_insn "*aarch64_mls_elt<mode>"
912 [(set (match_operand:VDQHS 0 "register_operand" "=w")
913 (minus:VDQHS
914 (match_operand:VDQHS 4 "register_operand" "0")
915 (mult:VDQHS
916 (vec_duplicate:VDQHS
917 (vec_select:<VEL>
918 (match_operand:VDQHS 1 "register_operand" "<h_con>")
919 (parallel [(match_operand:SI 2 "immediate_operand")])))
920 (match_operand:VDQHS 3 "register_operand" "w"))))]
921 "TARGET_SIMD"
922 {
923 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
924 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
925 }
926 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
927 )
928
929 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
930 [(set (match_operand:VDQHS 0 "register_operand" "=w")
931 (minus:VDQHS
932 (match_operand:VDQHS 4 "register_operand" "0")
933 (mult:VDQHS
934 (vec_duplicate:VDQHS
935 (vec_select:<VEL>
936 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
937 (parallel [(match_operand:SI 2 "immediate_operand")])))
938 (match_operand:VDQHS 3 "register_operand" "w"))))]
939 "TARGET_SIMD"
940 {
941 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
942 INTVAL (operands[2])));
943 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
944 }
945 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
946 )
947
948 ;; Max/Min operations.
949 (define_insn "<su><maxmin><mode>3"
950 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
951 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
952 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
953 "TARGET_SIMD"
954 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
955 [(set_attr "type" "neon_minmax<q>")]
956 )
957
958 (define_expand "<su><maxmin>v2di3"
959 [(set (match_operand:V2DI 0 "register_operand" "")
960 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
961 (match_operand:V2DI 2 "register_operand" "")))]
962 "TARGET_SIMD"
963 {
964 enum rtx_code cmp_operator;
965 rtx cmp_fmt;
966
967 switch (<CODE>)
968 {
969 case UMIN:
970 cmp_operator = LTU;
971 break;
972 case SMIN:
973 cmp_operator = LT;
974 break;
975 case UMAX:
976 cmp_operator = GTU;
977 break;
978 case SMAX:
979 cmp_operator = GT;
980 break;
981 default:
982 gcc_unreachable ();
983 }
984
985 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
986 emit_insn (gen_aarch64_vcond_internalv2div2di (operands[0], operands[1],
987 operands[2], cmp_fmt, operands[1], operands[2]));
988 DONE;
989 })
990
991 ;; Pairwise Integer Max/Min operations.
992 (define_insn "aarch64_<maxmin_uns>p<mode>"
993 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
994 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
995 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
996 MAXMINV))]
997 "TARGET_SIMD"
998 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
999 [(set_attr "type" "neon_minmax<q>")]
1000 )
1001
1002 ;; Pairwise FP Max/Min operations.
1003 (define_insn "aarch64_<maxmin_uns>p<mode>"
1004 [(set (match_operand:VDQF 0 "register_operand" "=w")
1005 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1006 (match_operand:VDQF 2 "register_operand" "w")]
1007 FMAXMINV))]
1008 "TARGET_SIMD"
1009 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1010 [(set_attr "type" "neon_minmax<q>")]
1011 )
1012
1013 ;; vec_concat gives a new vector with the low elements from operand 1, and
1014 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1015 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1016 ;; What that means, is that the RTL descriptions of the below patterns
1017 ;; need to change depending on endianness.
1018
1019 ;; Move to the low architectural bits of the register.
1020 ;; On little-endian this is { operand, zeroes }
1021 ;; On big-endian this is { zeroes, operand }
1022
1023 (define_insn "move_lo_quad_internal_<mode>"
1024 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1025 (vec_concat:VQ_NO2E
1026 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1027 (vec_duplicate:<VHALF> (const_int 0))))]
1028 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1029 "@
1030 dup\\t%d0, %1.d[0]
1031 fmov\\t%d0, %1
1032 dup\\t%d0, %1"
1033 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1034 (set_attr "simd" "yes,*,yes")
1035 (set_attr "fp" "*,yes,*")
1036 (set_attr "length" "4")]
1037 )
1038
1039 (define_insn "move_lo_quad_internal_<mode>"
1040 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1041 (vec_concat:VQ_2E
1042 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1043 (const_int 0)))]
1044 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1045 "@
1046 dup\\t%d0, %1.d[0]
1047 fmov\\t%d0, %1
1048 dup\\t%d0, %1"
1049 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1050 (set_attr "simd" "yes,*,yes")
1051 (set_attr "fp" "*,yes,*")
1052 (set_attr "length" "4")]
1053 )
1054
1055 (define_insn "move_lo_quad_internal_be_<mode>"
1056 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1057 (vec_concat:VQ_NO2E
1058 (vec_duplicate:<VHALF> (const_int 0))
1059 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1060 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1061 "@
1062 dup\\t%d0, %1.d[0]
1063 fmov\\t%d0, %1
1064 dup\\t%d0, %1"
1065 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1066 (set_attr "simd" "yes,*,yes")
1067 (set_attr "fp" "*,yes,*")
1068 (set_attr "length" "4")]
1069 )
1070
1071 (define_insn "move_lo_quad_internal_be_<mode>"
1072 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1073 (vec_concat:VQ_2E
1074 (const_int 0)
1075 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1076 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1077 "@
1078 dup\\t%d0, %1.d[0]
1079 fmov\\t%d0, %1
1080 dup\\t%d0, %1"
1081 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1082 (set_attr "simd" "yes,*,yes")
1083 (set_attr "fp" "*,yes,*")
1084 (set_attr "length" "4")]
1085 )
1086
1087 (define_expand "move_lo_quad_<mode>"
1088 [(match_operand:VQ 0 "register_operand")
1089 (match_operand:VQ 1 "register_operand")]
1090 "TARGET_SIMD"
1091 {
1092 if (BYTES_BIG_ENDIAN)
1093 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1094 else
1095 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1096 DONE;
1097 }
1098 )
1099
1100 ;; Move operand1 to the high architectural bits of the register, keeping
1101 ;; the low architectural bits of operand2.
1102 ;; For little-endian this is { operand2, operand1 }
1103 ;; For big-endian this is { operand1, operand2 }
1104
1105 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1106 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1107 (vec_concat:VQ
1108 (vec_select:<VHALF>
1109 (match_dup 0)
1110 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1111 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1112 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1113 "@
1114 ins\\t%0.d[1], %1.d[0]
1115 ins\\t%0.d[1], %1"
1116 [(set_attr "type" "neon_ins")]
1117 )
1118
1119 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1120 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1121 (vec_concat:VQ
1122 (match_operand:<VHALF> 1 "register_operand" "w,r")
1123 (vec_select:<VHALF>
1124 (match_dup 0)
1125 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1126 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1127 "@
1128 ins\\t%0.d[1], %1.d[0]
1129 ins\\t%0.d[1], %1"
1130 [(set_attr "type" "neon_ins")]
1131 )
1132
1133 (define_expand "move_hi_quad_<mode>"
1134 [(match_operand:VQ 0 "register_operand" "")
1135 (match_operand:<VHALF> 1 "register_operand" "")]
1136 "TARGET_SIMD"
1137 {
1138 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1139 if (BYTES_BIG_ENDIAN)
1140 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1141 operands[1], p));
1142 else
1143 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1144 operands[1], p));
1145 DONE;
1146 })
1147
1148 ;; Narrowing operations.
1149
1150 ;; For doubles.
1151 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1152 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1153 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1154 "TARGET_SIMD"
1155 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1156 [(set_attr "type" "neon_shift_imm_narrow_q")]
1157 )
1158
1159 (define_expand "vec_pack_trunc_<mode>"
1160 [(match_operand:<VNARROWD> 0 "register_operand" "")
1161 (match_operand:VDN 1 "register_operand" "")
1162 (match_operand:VDN 2 "register_operand" "")]
1163 "TARGET_SIMD"
1164 {
1165 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1166 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1167 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1168
1169 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1170 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1171 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1172 DONE;
1173 })
1174
1175 ;; For quads.
1176
1177 (define_insn "vec_pack_trunc_<mode>"
1178 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1179 (vec_concat:<VNARROWQ2>
1180 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1181 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1182 "TARGET_SIMD"
1183 {
1184 if (BYTES_BIG_ENDIAN)
1185 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1186 else
1187 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1188 }
1189 [(set_attr "type" "multiple")
1190 (set_attr "length" "8")]
1191 )
1192
1193 ;; Widening operations.
1194
1195 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1196 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1197 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1198 (match_operand:VQW 1 "register_operand" "w")
1199 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1200 )))]
1201 "TARGET_SIMD"
1202 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1203 [(set_attr "type" "neon_shift_imm_long")]
1204 )
1205
1206 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1207 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1208 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1209 (match_operand:VQW 1 "register_operand" "w")
1210 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1211 )))]
1212 "TARGET_SIMD"
1213 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1214 [(set_attr "type" "neon_shift_imm_long")]
1215 )
1216
1217 (define_expand "vec_unpack<su>_hi_<mode>"
1218 [(match_operand:<VWIDE> 0 "register_operand" "")
1219 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1220 "TARGET_SIMD"
1221 {
1222 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1223 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1224 operands[1], p));
1225 DONE;
1226 }
1227 )
1228
1229 (define_expand "vec_unpack<su>_lo_<mode>"
1230 [(match_operand:<VWIDE> 0 "register_operand" "")
1231 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1232 "TARGET_SIMD"
1233 {
1234 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1235 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1236 operands[1], p));
1237 DONE;
1238 }
1239 )
1240
1241 ;; Widening arithmetic.
1242
1243 (define_insn "*aarch64_<su>mlal_lo<mode>"
1244 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1245 (plus:<VWIDE>
1246 (mult:<VWIDE>
1247 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1248 (match_operand:VQW 2 "register_operand" "w")
1249 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1250 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1251 (match_operand:VQW 4 "register_operand" "w")
1252 (match_dup 3))))
1253 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1254 "TARGET_SIMD"
1255 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1256 [(set_attr "type" "neon_mla_<Vetype>_long")]
1257 )
1258
1259 (define_insn "*aarch64_<su>mlal_hi<mode>"
1260 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1261 (plus:<VWIDE>
1262 (mult:<VWIDE>
1263 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1264 (match_operand:VQW 2 "register_operand" "w")
1265 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1266 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1267 (match_operand:VQW 4 "register_operand" "w")
1268 (match_dup 3))))
1269 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1270 "TARGET_SIMD"
1271 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1272 [(set_attr "type" "neon_mla_<Vetype>_long")]
1273 )
1274
1275 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1276 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1277 (minus:<VWIDE>
1278 (match_operand:<VWIDE> 1 "register_operand" "0")
1279 (mult:<VWIDE>
1280 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1281 (match_operand:VQW 2 "register_operand" "w")
1282 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1283 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1284 (match_operand:VQW 4 "register_operand" "w")
1285 (match_dup 3))))))]
1286 "TARGET_SIMD"
1287 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1288 [(set_attr "type" "neon_mla_<Vetype>_long")]
1289 )
1290
1291 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1292 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1293 (minus:<VWIDE>
1294 (match_operand:<VWIDE> 1 "register_operand" "0")
1295 (mult:<VWIDE>
1296 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1297 (match_operand:VQW 2 "register_operand" "w")
1298 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1299 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1300 (match_operand:VQW 4 "register_operand" "w")
1301 (match_dup 3))))))]
1302 "TARGET_SIMD"
1303 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1304 [(set_attr "type" "neon_mla_<Vetype>_long")]
1305 )
1306
1307 (define_insn "*aarch64_<su>mlal<mode>"
1308 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1309 (plus:<VWIDE>
1310 (mult:<VWIDE>
1311 (ANY_EXTEND:<VWIDE>
1312 (match_operand:VD_BHSI 1 "register_operand" "w"))
1313 (ANY_EXTEND:<VWIDE>
1314 (match_operand:VD_BHSI 2 "register_operand" "w")))
1315 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1316 "TARGET_SIMD"
1317 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1318 [(set_attr "type" "neon_mla_<Vetype>_long")]
1319 )
1320
1321 (define_insn "*aarch64_<su>mlsl<mode>"
1322 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1323 (minus:<VWIDE>
1324 (match_operand:<VWIDE> 1 "register_operand" "0")
1325 (mult:<VWIDE>
1326 (ANY_EXTEND:<VWIDE>
1327 (match_operand:VD_BHSI 2 "register_operand" "w"))
1328 (ANY_EXTEND:<VWIDE>
1329 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1330 "TARGET_SIMD"
1331 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1332 [(set_attr "type" "neon_mla_<Vetype>_long")]
1333 )
1334
1335 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1336 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1337 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1338 (match_operand:VQW 1 "register_operand" "w")
1339 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1340 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1341 (match_operand:VQW 2 "register_operand" "w")
1342 (match_dup 3)))))]
1343 "TARGET_SIMD"
1344 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1345 [(set_attr "type" "neon_mul_<Vetype>_long")]
1346 )
1347
1348 (define_expand "vec_widen_<su>mult_lo_<mode>"
1349 [(match_operand:<VWIDE> 0 "register_operand" "")
1350 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1351 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1352 "TARGET_SIMD"
1353 {
1354 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1355 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1356 operands[1],
1357 operands[2], p));
1358 DONE;
1359 }
1360 )
1361
1362 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1363 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1364 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1365 (match_operand:VQW 1 "register_operand" "w")
1366 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1367 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1368 (match_operand:VQW 2 "register_operand" "w")
1369 (match_dup 3)))))]
1370 "TARGET_SIMD"
1371 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1372 [(set_attr "type" "neon_mul_<Vetype>_long")]
1373 )
1374
1375 (define_expand "vec_widen_<su>mult_hi_<mode>"
1376 [(match_operand:<VWIDE> 0 "register_operand" "")
1377 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1378 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1379 "TARGET_SIMD"
1380 {
1381 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1382 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1383 operands[1],
1384 operands[2], p));
1385 DONE;
1386
1387 }
1388 )
1389
1390 ;; FP vector operations.
1391 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1392 ;; double-precision (64-bit) floating-point data types and arithmetic as
1393 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1394 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1395 ;;
1396 ;; Floating-point operations can raise an exception. Vectorizing such
1397 ;; operations are safe because of reasons explained below.
1398 ;;
1399 ;; ARMv8 permits an extension to enable trapped floating-point
1400 ;; exception handling, however this is an optional feature. In the
1401 ;; event of a floating-point exception being raised by vectorised
1402 ;; code then:
1403 ;; 1. If trapped floating-point exceptions are available, then a trap
1404 ;; will be taken when any lane raises an enabled exception. A trap
1405 ;; handler may determine which lane raised the exception.
1406 ;; 2. Alternatively a sticky exception flag is set in the
1407 ;; floating-point status register (FPSR). Software may explicitly
1408 ;; test the exception flags, in which case the tests will either
1409 ;; prevent vectorisation, allowing precise identification of the
1410 ;; failing operation, or if tested outside of vectorisable regions
1411 ;; then the specific operation and lane are not of interest.
1412
1413 ;; FP arithmetic operations.
1414
1415 (define_insn "add<mode>3"
1416 [(set (match_operand:VDQF 0 "register_operand" "=w")
1417 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1418 (match_operand:VDQF 2 "register_operand" "w")))]
1419 "TARGET_SIMD"
1420 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1421 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1422 )
1423
1424 (define_insn "sub<mode>3"
1425 [(set (match_operand:VDQF 0 "register_operand" "=w")
1426 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1427 (match_operand:VDQF 2 "register_operand" "w")))]
1428 "TARGET_SIMD"
1429 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1430 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1431 )
1432
1433 (define_insn "mul<mode>3"
1434 [(set (match_operand:VDQF 0 "register_operand" "=w")
1435 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1436 (match_operand:VDQF 2 "register_operand" "w")))]
1437 "TARGET_SIMD"
1438 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1439 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1440 )
1441
1442 (define_insn "div<mode>3"
1443 [(set (match_operand:VDQF 0 "register_operand" "=w")
1444 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1445 (match_operand:VDQF 2 "register_operand" "w")))]
1446 "TARGET_SIMD"
1447 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1448 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1449 )
1450
1451 (define_insn "neg<mode>2"
1452 [(set (match_operand:VDQF 0 "register_operand" "=w")
1453 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1454 "TARGET_SIMD"
1455 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1456 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1457 )
1458
1459 (define_insn "abs<mode>2"
1460 [(set (match_operand:VDQF 0 "register_operand" "=w")
1461 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1462 "TARGET_SIMD"
1463 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1464 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1465 )
1466
1467 (define_insn "fma<mode>4"
1468 [(set (match_operand:VDQF 0 "register_operand" "=w")
1469 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1470 (match_operand:VDQF 2 "register_operand" "w")
1471 (match_operand:VDQF 3 "register_operand" "0")))]
1472 "TARGET_SIMD"
1473 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1474 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1475 )
1476
1477 (define_insn "*aarch64_fma4_elt<mode>"
1478 [(set (match_operand:VDQF 0 "register_operand" "=w")
1479 (fma:VDQF
1480 (vec_duplicate:VDQF
1481 (vec_select:<VEL>
1482 (match_operand:VDQF 1 "register_operand" "<h_con>")
1483 (parallel [(match_operand:SI 2 "immediate_operand")])))
1484 (match_operand:VDQF 3 "register_operand" "w")
1485 (match_operand:VDQF 4 "register_operand" "0")))]
1486 "TARGET_SIMD"
1487 {
1488 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1489 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1490 }
1491 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1492 )
1493
1494 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1495 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1496 (fma:VDQSF
1497 (vec_duplicate:VDQSF
1498 (vec_select:<VEL>
1499 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1500 (parallel [(match_operand:SI 2 "immediate_operand")])))
1501 (match_operand:VDQSF 3 "register_operand" "w")
1502 (match_operand:VDQSF 4 "register_operand" "0")))]
1503 "TARGET_SIMD"
1504 {
1505 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1506 INTVAL (operands[2])));
1507 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1508 }
1509 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1510 )
1511
1512 (define_insn "*aarch64_fma4_elt_to_128df"
1513 [(set (match_operand:V2DF 0 "register_operand" "=w")
1514 (fma:V2DF
1515 (vec_duplicate:V2DF
1516 (match_operand:DF 1 "register_operand" "w"))
1517 (match_operand:V2DF 2 "register_operand" "w")
1518 (match_operand:V2DF 3 "register_operand" "0")))]
1519 "TARGET_SIMD"
1520 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1521 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1522 )
1523
1524 (define_insn "*aarch64_fma4_elt_to_64v2df"
1525 [(set (match_operand:DF 0 "register_operand" "=w")
1526 (fma:DF
1527 (vec_select:DF
1528 (match_operand:V2DF 1 "register_operand" "w")
1529 (parallel [(match_operand:SI 2 "immediate_operand")]))
1530 (match_operand:DF 3 "register_operand" "w")
1531 (match_operand:DF 4 "register_operand" "0")))]
1532 "TARGET_SIMD"
1533 {
1534 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1535 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1536 }
1537 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1538 )
1539
1540 (define_insn "fnma<mode>4"
1541 [(set (match_operand:VDQF 0 "register_operand" "=w")
1542 (fma:VDQF
1543 (match_operand:VDQF 1 "register_operand" "w")
1544 (neg:VDQF
1545 (match_operand:VDQF 2 "register_operand" "w"))
1546 (match_operand:VDQF 3 "register_operand" "0")))]
1547 "TARGET_SIMD"
1548 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1549 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1550 )
1551
1552 (define_insn "*aarch64_fnma4_elt<mode>"
1553 [(set (match_operand:VDQF 0 "register_operand" "=w")
1554 (fma:VDQF
1555 (neg:VDQF
1556 (match_operand:VDQF 3 "register_operand" "w"))
1557 (vec_duplicate:VDQF
1558 (vec_select:<VEL>
1559 (match_operand:VDQF 1 "register_operand" "<h_con>")
1560 (parallel [(match_operand:SI 2 "immediate_operand")])))
1561 (match_operand:VDQF 4 "register_operand" "0")))]
1562 "TARGET_SIMD"
1563 {
1564 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1565 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1566 }
1567 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1568 )
1569
1570 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1571 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1572 (fma:VDQSF
1573 (neg:VDQSF
1574 (match_operand:VDQSF 3 "register_operand" "w"))
1575 (vec_duplicate:VDQSF
1576 (vec_select:<VEL>
1577 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1578 (parallel [(match_operand:SI 2 "immediate_operand")])))
1579 (match_operand:VDQSF 4 "register_operand" "0")))]
1580 "TARGET_SIMD"
1581 {
1582 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1583 INTVAL (operands[2])));
1584 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1585 }
1586 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1587 )
1588
1589 (define_insn "*aarch64_fnma4_elt_to_128df"
1590 [(set (match_operand:V2DF 0 "register_operand" "=w")
1591 (fma:V2DF
1592 (neg:V2DF
1593 (match_operand:V2DF 2 "register_operand" "w"))
1594 (vec_duplicate:V2DF
1595 (match_operand:DF 1 "register_operand" "w"))
1596 (match_operand:V2DF 3 "register_operand" "0")))]
1597 "TARGET_SIMD"
1598 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1599 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1600 )
1601
1602 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1603 [(set (match_operand:DF 0 "register_operand" "=w")
1604 (fma:DF
1605 (vec_select:DF
1606 (match_operand:V2DF 1 "register_operand" "w")
1607 (parallel [(match_operand:SI 2 "immediate_operand")]))
1608 (neg:DF
1609 (match_operand:DF 3 "register_operand" "w"))
1610 (match_operand:DF 4 "register_operand" "0")))]
1611 "TARGET_SIMD"
1612 {
1613 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1614 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1615 }
1616 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1617 )
1618
1619 ;; Vector versions of the floating-point frint patterns.
1620 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1621 (define_insn "<frint_pattern><mode>2"
1622 [(set (match_operand:VDQF 0 "register_operand" "=w")
1623 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1624 FRINT))]
1625 "TARGET_SIMD"
1626 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1627 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1628 )
1629
1630 ;; Vector versions of the fcvt standard patterns.
1631 ;; Expands to lbtrunc, lround, lceil, lfloor
1632 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1633 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1634 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1635 [(match_operand:VDQF 1 "register_operand" "w")]
1636 FCVT)))]
1637 "TARGET_SIMD"
1638 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1639 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1640 )
1641
1642 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1643 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1644 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1645 [(match_operand:VDQF 1 "register_operand")]
1646 UNSPEC_FRINTZ)))]
1647 "TARGET_SIMD"
1648 {})
1649
1650 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1651 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1652 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1653 [(match_operand:VDQF 1 "register_operand")]
1654 UNSPEC_FRINTZ)))]
1655 "TARGET_SIMD"
1656 {})
1657
1658 (define_expand "ftrunc<VDQF:mode>2"
1659 [(set (match_operand:VDQF 0 "register_operand")
1660 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1661 UNSPEC_FRINTZ))]
1662 "TARGET_SIMD"
1663 {})
1664
1665 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1666 [(set (match_operand:VDQF 0 "register_operand" "=w")
1667 (FLOATUORS:VDQF
1668 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1669 "TARGET_SIMD"
1670 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1671 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1672 )
1673
1674 ;; Conversions between vectors of floats and doubles.
1675 ;; Contains a mix of patterns to match standard pattern names
1676 ;; and those for intrinsics.
1677
1678 ;; Float widening operations.
1679
1680 (define_insn "vec_unpacks_lo_v4sf"
1681 [(set (match_operand:V2DF 0 "register_operand" "=w")
1682 (float_extend:V2DF
1683 (vec_select:V2SF
1684 (match_operand:V4SF 1 "register_operand" "w")
1685 (parallel [(const_int 0) (const_int 1)])
1686 )))]
1687 "TARGET_SIMD"
1688 "fcvtl\\t%0.2d, %1.2s"
1689 [(set_attr "type" "neon_fp_cvt_widen_s")]
1690 )
1691
1692 (define_insn "aarch64_float_extend_lo_v2df"
1693 [(set (match_operand:V2DF 0 "register_operand" "=w")
1694 (float_extend:V2DF
1695 (match_operand:V2SF 1 "register_operand" "w")))]
1696 "TARGET_SIMD"
1697 "fcvtl\\t%0.2d, %1.2s"
1698 [(set_attr "type" "neon_fp_cvt_widen_s")]
1699 )
1700
1701 (define_insn "vec_unpacks_hi_v4sf"
1702 [(set (match_operand:V2DF 0 "register_operand" "=w")
1703 (float_extend:V2DF
1704 (vec_select:V2SF
1705 (match_operand:V4SF 1 "register_operand" "w")
1706 (parallel [(const_int 2) (const_int 3)])
1707 )))]
1708 "TARGET_SIMD"
1709 "fcvtl2\\t%0.2d, %1.4s"
1710 [(set_attr "type" "neon_fp_cvt_widen_s")]
1711 )
1712
1713 ;; Float narrowing operations.
1714
1715 (define_insn "aarch64_float_truncate_lo_v2sf"
1716 [(set (match_operand:V2SF 0 "register_operand" "=w")
1717 (float_truncate:V2SF
1718 (match_operand:V2DF 1 "register_operand" "w")))]
1719 "TARGET_SIMD"
1720 "fcvtn\\t%0.2s, %1.2d"
1721 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1722 )
1723
1724 (define_insn "aarch64_float_truncate_hi_v4sf"
1725 [(set (match_operand:V4SF 0 "register_operand" "=w")
1726 (vec_concat:V4SF
1727 (match_operand:V2SF 1 "register_operand" "0")
1728 (float_truncate:V2SF
1729 (match_operand:V2DF 2 "register_operand" "w"))))]
1730 "TARGET_SIMD"
1731 "fcvtn2\\t%0.4s, %2.2d"
1732 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1733 )
1734
1735 (define_expand "vec_pack_trunc_v2df"
1736 [(set (match_operand:V4SF 0 "register_operand")
1737 (vec_concat:V4SF
1738 (float_truncate:V2SF
1739 (match_operand:V2DF 1 "register_operand"))
1740 (float_truncate:V2SF
1741 (match_operand:V2DF 2 "register_operand"))
1742 ))]
1743 "TARGET_SIMD"
1744 {
1745 rtx tmp = gen_reg_rtx (V2SFmode);
1746 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1747 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1748
1749 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1750 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1751 tmp, operands[hi]));
1752 DONE;
1753 }
1754 )
1755
1756 (define_expand "vec_pack_trunc_df"
1757 [(set (match_operand:V2SF 0 "register_operand")
1758 (vec_concat:V2SF
1759 (float_truncate:SF
1760 (match_operand:DF 1 "register_operand"))
1761 (float_truncate:SF
1762 (match_operand:DF 2 "register_operand"))
1763 ))]
1764 "TARGET_SIMD"
1765 {
1766 rtx tmp = gen_reg_rtx (V2SFmode);
1767 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1768 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1769
1770 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1771 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1772 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1773 DONE;
1774 }
1775 )
1776
1777 (define_insn "aarch64_vmls<mode>"
1778 [(set (match_operand:VDQF 0 "register_operand" "=w")
1779 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1780 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1781 (match_operand:VDQF 3 "register_operand" "w"))))]
1782 "TARGET_SIMD"
1783 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1784 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1785 )
1786
1787 ;; FP Max/Min
1788 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1789 ;; expression like:
1790 ;; a = (b < c) ? b : c;
1791 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1792 ;; either explicitly or indirectly via -ffast-math.
1793 ;;
1794 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1795 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1796 ;; operand will be returned when both operands are zero (i.e. they may not
1797 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1798 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1799 ;; NaNs.
1800
1801 (define_insn "<su><maxmin><mode>3"
1802 [(set (match_operand:VDQF 0 "register_operand" "=w")
1803 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1804 (match_operand:VDQF 2 "register_operand" "w")))]
1805 "TARGET_SIMD"
1806 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1807 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1808 )
1809
1810 (define_insn "<maxmin_uns><mode>3"
1811 [(set (match_operand:VDQF 0 "register_operand" "=w")
1812 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1813 (match_operand:VDQF 2 "register_operand" "w")]
1814 FMAXMIN_UNS))]
1815 "TARGET_SIMD"
1816 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1817 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1818 )
1819
1820 ;; 'across lanes' add.
1821
1822 (define_expand "reduc_plus_scal_<mode>"
1823 [(match_operand:<VEL> 0 "register_operand" "=w")
1824 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
1825 UNSPEC_ADDV)]
1826 "TARGET_SIMD"
1827 {
1828 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1829 rtx scratch = gen_reg_rtx (<MODE>mode);
1830 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1831 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1832 DONE;
1833 }
1834 )
1835
1836 (define_expand "reduc_plus_scal_<mode>"
1837 [(match_operand:<VEL> 0 "register_operand" "=w")
1838 (match_operand:V2F 1 "register_operand" "w")]
1839 "TARGET_SIMD"
1840 {
1841 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1842 rtx scratch = gen_reg_rtx (<MODE>mode);
1843 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1844 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1845 DONE;
1846 }
1847 )
1848
1849 (define_insn "aarch64_reduc_plus_internal<mode>"
1850 [(set (match_operand:VDQV 0 "register_operand" "=w")
1851 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1852 UNSPEC_ADDV))]
1853 "TARGET_SIMD"
1854 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
1855 [(set_attr "type" "neon_reduc_add<q>")]
1856 )
1857
1858 (define_insn "aarch64_reduc_plus_internalv2si"
1859 [(set (match_operand:V2SI 0 "register_operand" "=w")
1860 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1861 UNSPEC_ADDV))]
1862 "TARGET_SIMD"
1863 "addp\\t%0.2s, %1.2s, %1.2s"
1864 [(set_attr "type" "neon_reduc_add")]
1865 )
1866
1867 (define_insn "aarch64_reduc_plus_internal<mode>"
1868 [(set (match_operand:V2F 0 "register_operand" "=w")
1869 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1870 UNSPEC_FADDV))]
1871 "TARGET_SIMD"
1872 "faddp\\t%<Vetype>0, %1.<Vtype>"
1873 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
1874 )
1875
1876 (define_insn "aarch64_addpv4sf"
1877 [(set (match_operand:V4SF 0 "register_operand" "=w")
1878 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1879 UNSPEC_FADDV))]
1880 "TARGET_SIMD"
1881 "faddp\\t%0.4s, %1.4s, %1.4s"
1882 [(set_attr "type" "neon_fp_reduc_add_s_q")]
1883 )
1884
1885 (define_expand "reduc_plus_scal_v4sf"
1886 [(set (match_operand:SF 0 "register_operand")
1887 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1888 UNSPEC_FADDV))]
1889 "TARGET_SIMD"
1890 {
1891 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
1892 rtx scratch = gen_reg_rtx (V4SFmode);
1893 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
1894 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
1895 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
1896 DONE;
1897 })
1898
1899 (define_insn "clrsb<mode>2"
1900 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1901 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1902 "TARGET_SIMD"
1903 "cls\\t%0.<Vtype>, %1.<Vtype>"
1904 [(set_attr "type" "neon_cls<q>")]
1905 )
1906
1907 (define_insn "clz<mode>2"
1908 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1909 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1910 "TARGET_SIMD"
1911 "clz\\t%0.<Vtype>, %1.<Vtype>"
1912 [(set_attr "type" "neon_cls<q>")]
1913 )
1914
1915 (define_insn "popcount<mode>2"
1916 [(set (match_operand:VB 0 "register_operand" "=w")
1917 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
1918 "TARGET_SIMD"
1919 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
1920 [(set_attr "type" "neon_cnt<q>")]
1921 )
1922
1923 ;; 'across lanes' max and min ops.
1924
1925 ;; Template for outputting a scalar, so we can create __builtins which can be
1926 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
1927 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1928 [(match_operand:<VEL> 0 "register_operand")
1929 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1930 FMAXMINV)]
1931 "TARGET_SIMD"
1932 {
1933 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1934 rtx scratch = gen_reg_rtx (<MODE>mode);
1935 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1936 operands[1]));
1937 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1938 DONE;
1939 }
1940 )
1941
1942 ;; Likewise for integer cases, signed and unsigned.
1943 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1944 [(match_operand:<VEL> 0 "register_operand")
1945 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
1946 MAXMINV)]
1947 "TARGET_SIMD"
1948 {
1949 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1950 rtx scratch = gen_reg_rtx (<MODE>mode);
1951 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1952 operands[1]));
1953 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1954 DONE;
1955 }
1956 )
1957
1958 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1959 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
1960 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
1961 MAXMINV))]
1962 "TARGET_SIMD"
1963 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1964 [(set_attr "type" "neon_reduc_minmax<q>")]
1965 )
1966
1967 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
1968 [(set (match_operand:V2SI 0 "register_operand" "=w")
1969 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1970 MAXMINV))]
1971 "TARGET_SIMD"
1972 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1973 [(set_attr "type" "neon_reduc_minmax")]
1974 )
1975
1976 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1977 [(set (match_operand:VDQF 0 "register_operand" "=w")
1978 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1979 FMAXMINV))]
1980 "TARGET_SIMD"
1981 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
1982 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
1983 )
1984
1985 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1986 ;; allocation.
1987 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1988 ;; to select.
1989 ;;
1990 ;; Thus our BSL is of the form:
1991 ;; op0 = bsl (mask, op2, op3)
1992 ;; We can use any of:
1993 ;;
1994 ;; if (op0 = mask)
1995 ;; bsl mask, op1, op2
1996 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1997 ;; bit op0, op2, mask
1998 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1999 ;; bif op0, op1, mask
2000
2001 (define_insn "aarch64_simd_bsl<mode>_internal"
2002 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2003 (xor:VSDQ_I_DI
2004 (and:VSDQ_I_DI
2005 (xor:VSDQ_I_DI
2006 (match_operand:<V_cmp_result> 3 "register_operand" "w,0,w")
2007 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2008 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2009 (match_dup:<V_cmp_result> 3)
2010 ))]
2011 "TARGET_SIMD"
2012 "@
2013 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2014 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2015 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2016 [(set_attr "type" "neon_bsl<q>")]
2017 )
2018
2019 (define_expand "aarch64_simd_bsl<mode>"
2020 [(match_operand:VALLDIF 0 "register_operand")
2021 (match_operand:<V_cmp_result> 1 "register_operand")
2022 (match_operand:VALLDIF 2 "register_operand")
2023 (match_operand:VALLDIF 3 "register_operand")]
2024 "TARGET_SIMD"
2025 {
2026 /* We can't alias operands together if they have different modes. */
2027 rtx tmp = operands[0];
2028 if (FLOAT_MODE_P (<MODE>mode))
2029 {
2030 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2031 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2032 tmp = gen_reg_rtx (<V_cmp_result>mode);
2033 }
2034 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
2035 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2036 operands[1],
2037 operands[2],
2038 operands[3]));
2039 if (tmp != operands[0])
2040 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2041
2042 DONE;
2043 })
2044
2045 (define_expand "aarch64_vcond_internal<mode><mode>"
2046 [(set (match_operand:VDQ_I 0 "register_operand")
2047 (if_then_else:VDQ_I
2048 (match_operator 3 "comparison_operator"
2049 [(match_operand:VDQ_I 4 "register_operand")
2050 (match_operand:VDQ_I 5 "nonmemory_operand")])
2051 (match_operand:VDQ_I 1 "nonmemory_operand")
2052 (match_operand:VDQ_I 2 "nonmemory_operand")))]
2053 "TARGET_SIMD"
2054 {
2055 rtx op1 = operands[1];
2056 rtx op2 = operands[2];
2057 rtx mask = gen_reg_rtx (<MODE>mode);
2058 enum rtx_code code = GET_CODE (operands[3]);
2059
2060 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
2061 and desirable for other comparisons if it results in FOO ? -1 : 0
2062 (this allows direct use of the comparison result without a bsl). */
2063 if (code == NE
2064 || (code != EQ
2065 && op1 == CONST0_RTX (<V_cmp_result>mode)
2066 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
2067 {
2068 op1 = operands[2];
2069 op2 = operands[1];
2070 switch (code)
2071 {
2072 case LE: code = GT; break;
2073 case LT: code = GE; break;
2074 case GE: code = LT; break;
2075 case GT: code = LE; break;
2076 /* No case EQ. */
2077 case NE: code = EQ; break;
2078 case LTU: code = GEU; break;
2079 case LEU: code = GTU; break;
2080 case GTU: code = LEU; break;
2081 case GEU: code = LTU; break;
2082 default: gcc_unreachable ();
2083 }
2084 }
2085
2086 /* Make sure we can handle the last operand. */
2087 switch (code)
2088 {
2089 case NE:
2090 /* Normalized to EQ above. */
2091 gcc_unreachable ();
2092
2093 case LE:
2094 case LT:
2095 case GE:
2096 case GT:
2097 case EQ:
2098 /* These instructions have a form taking an immediate zero. */
2099 if (operands[5] == CONST0_RTX (<MODE>mode))
2100 break;
2101 /* Fall through, as may need to load into register. */
2102 default:
2103 if (!REG_P (operands[5]))
2104 operands[5] = force_reg (<MODE>mode, operands[5]);
2105 break;
2106 }
2107
2108 switch (code)
2109 {
2110 case LT:
2111 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2112 break;
2113
2114 case GE:
2115 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2116 break;
2117
2118 case LE:
2119 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2120 break;
2121
2122 case GT:
2123 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2124 break;
2125
2126 case LTU:
2127 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2128 break;
2129
2130 case GEU:
2131 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2132 break;
2133
2134 case LEU:
2135 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2136 break;
2137
2138 case GTU:
2139 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2140 break;
2141
2142 /* NE has been normalized to EQ above. */
2143 case EQ:
2144 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2145 break;
2146
2147 default:
2148 gcc_unreachable ();
2149 }
2150
2151 /* If we have (a = (b CMP c) ? -1 : 0);
2152 Then we can simply move the generated mask. */
2153
2154 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2155 && op2 == CONST0_RTX (<V_cmp_result>mode))
2156 emit_move_insn (operands[0], mask);
2157 else
2158 {
2159 if (!REG_P (op1))
2160 op1 = force_reg (<MODE>mode, op1);
2161 if (!REG_P (op2))
2162 op2 = force_reg (<MODE>mode, op2);
2163 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2164 op1, op2));
2165 }
2166
2167 DONE;
2168 })
2169
2170 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2171 [(set (match_operand:VDQF_COND 0 "register_operand")
2172 (if_then_else:VDQF
2173 (match_operator 3 "comparison_operator"
2174 [(match_operand:VDQF 4 "register_operand")
2175 (match_operand:VDQF 5 "nonmemory_operand")])
2176 (match_operand:VDQF_COND 1 "nonmemory_operand")
2177 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2178 "TARGET_SIMD"
2179 {
2180 int inverse = 0;
2181 int use_zero_form = 0;
2182 int swap_bsl_operands = 0;
2183 rtx op1 = operands[1];
2184 rtx op2 = operands[2];
2185 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2186 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2187
2188 rtx (*base_comparison) (rtx, rtx, rtx);
2189 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2190
2191 switch (GET_CODE (operands[3]))
2192 {
2193 case GE:
2194 case GT:
2195 case LE:
2196 case LT:
2197 case EQ:
2198 if (operands[5] == CONST0_RTX (<MODE>mode))
2199 {
2200 use_zero_form = 1;
2201 break;
2202 }
2203 /* Fall through. */
2204 default:
2205 if (!REG_P (operands[5]))
2206 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2207 }
2208
2209 switch (GET_CODE (operands[3]))
2210 {
2211 case LT:
2212 case UNLT:
2213 inverse = 1;
2214 /* Fall through. */
2215 case GE:
2216 case UNGE:
2217 case ORDERED:
2218 case UNORDERED:
2219 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2220 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2221 break;
2222 case LE:
2223 case UNLE:
2224 inverse = 1;
2225 /* Fall through. */
2226 case GT:
2227 case UNGT:
2228 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2229 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2230 break;
2231 case EQ:
2232 case NE:
2233 case UNEQ:
2234 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2235 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2236 break;
2237 default:
2238 gcc_unreachable ();
2239 }
2240
2241 switch (GET_CODE (operands[3]))
2242 {
2243 case LT:
2244 case LE:
2245 case GT:
2246 case GE:
2247 case EQ:
2248 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2249 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2250 a GE b -> a GE b
2251 a GT b -> a GT b
2252 a LE b -> b GE a
2253 a LT b -> b GT a
2254 a EQ b -> a EQ b
2255 Note that there also exist direct comparison against 0 forms,
2256 so catch those as a special case. */
2257 if (use_zero_form)
2258 {
2259 inverse = 0;
2260 switch (GET_CODE (operands[3]))
2261 {
2262 case LT:
2263 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2264 break;
2265 case LE:
2266 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2267 break;
2268 default:
2269 /* Do nothing, other zero form cases already have the correct
2270 base_comparison. */
2271 break;
2272 }
2273 }
2274
2275 if (!inverse)
2276 emit_insn (base_comparison (mask, operands[4], operands[5]));
2277 else
2278 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2279 break;
2280 case UNLT:
2281 case UNLE:
2282 case UNGT:
2283 case UNGE:
2284 case NE:
2285 /* FCM returns false for lanes which are unordered, so if we use
2286 the inverse of the comparison we actually want to emit, then
2287 swap the operands to BSL, we will end up with the correct result.
2288 Note that a NE NaN and NaN NE b are true for all a, b.
2289
2290 Our transformations are:
2291 a GE b -> !(b GT a)
2292 a GT b -> !(b GE a)
2293 a LE b -> !(a GT b)
2294 a LT b -> !(a GE b)
2295 a NE b -> !(a EQ b) */
2296
2297 if (inverse)
2298 emit_insn (base_comparison (mask, operands[4], operands[5]));
2299 else
2300 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2301
2302 swap_bsl_operands = 1;
2303 break;
2304 case UNEQ:
2305 /* We check (a > b || b > a). combining these comparisons give us
2306 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2307 will then give us (a == b || a UNORDERED b) as intended. */
2308
2309 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2310 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2311 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2312 swap_bsl_operands = 1;
2313 break;
2314 case UNORDERED:
2315 /* Operands are ORDERED iff (a > b || b >= a).
2316 Swapping the operands to BSL will give the UNORDERED case. */
2317 swap_bsl_operands = 1;
2318 /* Fall through. */
2319 case ORDERED:
2320 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2321 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2322 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2323 break;
2324 default:
2325 gcc_unreachable ();
2326 }
2327
2328 if (swap_bsl_operands)
2329 {
2330 op1 = operands[2];
2331 op2 = operands[1];
2332 }
2333
2334 /* If we have (a = (b CMP c) ? -1 : 0);
2335 Then we can simply move the generated mask. */
2336
2337 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2338 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2339 emit_move_insn (operands[0], mask);
2340 else
2341 {
2342 if (!REG_P (op1))
2343 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2344 if (!REG_P (op2))
2345 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2346 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2347 op1, op2));
2348 }
2349
2350 DONE;
2351 })
2352
2353 (define_expand "vcond<mode><mode>"
2354 [(set (match_operand:VALL 0 "register_operand")
2355 (if_then_else:VALL
2356 (match_operator 3 "comparison_operator"
2357 [(match_operand:VALL 4 "register_operand")
2358 (match_operand:VALL 5 "nonmemory_operand")])
2359 (match_operand:VALL 1 "nonmemory_operand")
2360 (match_operand:VALL 2 "nonmemory_operand")))]
2361 "TARGET_SIMD"
2362 {
2363 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2364 operands[2], operands[3],
2365 operands[4], operands[5]));
2366 DONE;
2367 })
2368
2369 (define_expand "vcond<v_cmp_result><mode>"
2370 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2371 (if_then_else:<V_cmp_result>
2372 (match_operator 3 "comparison_operator"
2373 [(match_operand:VDQF 4 "register_operand")
2374 (match_operand:VDQF 5 "nonmemory_operand")])
2375 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2376 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2377 "TARGET_SIMD"
2378 {
2379 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2380 operands[0], operands[1],
2381 operands[2], operands[3],
2382 operands[4], operands[5]));
2383 DONE;
2384 })
2385
2386 (define_expand "vcondu<mode><mode>"
2387 [(set (match_operand:VDQ_I 0 "register_operand")
2388 (if_then_else:VDQ_I
2389 (match_operator 3 "comparison_operator"
2390 [(match_operand:VDQ_I 4 "register_operand")
2391 (match_operand:VDQ_I 5 "nonmemory_operand")])
2392 (match_operand:VDQ_I 1 "nonmemory_operand")
2393 (match_operand:VDQ_I 2 "nonmemory_operand")))]
2394 "TARGET_SIMD"
2395 {
2396 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2397 operands[2], operands[3],
2398 operands[4], operands[5]));
2399 DONE;
2400 })
2401
2402 ;; Patterns for AArch64 SIMD Intrinsics.
2403
2404 ;; Lane extraction with sign extension to general purpose register.
2405 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2406 [(set (match_operand:GPI 0 "register_operand" "=r")
2407 (sign_extend:GPI
2408 (vec_select:<VEL>
2409 (match_operand:VDQQH 1 "register_operand" "w")
2410 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2411 "TARGET_SIMD"
2412 {
2413 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2414 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2415 }
2416 [(set_attr "type" "neon_to_gp<q>")]
2417 )
2418
2419 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2420 [(set (match_operand:SI 0 "register_operand" "=r")
2421 (zero_extend:SI
2422 (vec_select:<VEL>
2423 (match_operand:VDQQH 1 "register_operand" "w")
2424 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2425 "TARGET_SIMD"
2426 {
2427 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2428 return "umov\\t%w0, %1.<Vetype>[%2]";
2429 }
2430 [(set_attr "type" "neon_to_gp<q>")]
2431 )
2432
2433 ;; Lane extraction of a value, neither sign nor zero extension
2434 ;; is guaranteed so upper bits should be considered undefined.
2435 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2436 (define_insn "aarch64_get_lane<mode>"
2437 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2438 (vec_select:<VEL>
2439 (match_operand:VALL 1 "register_operand" "w, w, w")
2440 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2441 "TARGET_SIMD"
2442 {
2443 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2444 switch (which_alternative)
2445 {
2446 case 0:
2447 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2448 case 1:
2449 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2450 case 2:
2451 return "st1\\t{%1.<Vetype>}[%2], %0";
2452 default:
2453 gcc_unreachable ();
2454 }
2455 }
2456 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2457 )
2458
2459 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2460 ;; dest vector.
2461
2462 (define_insn "*aarch64_combinez<mode>"
2463 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2464 (vec_concat:<VDBL>
2465 (match_operand:VD_BHSI 1 "register_operand" "w")
2466 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz")))]
2467 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2468 "mov\\t%0.8b, %1.8b"
2469 [(set_attr "type" "neon_move<q>")]
2470 )
2471
2472 (define_insn "*aarch64_combinez_be<mode>"
2473 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2474 (vec_concat:<VDBL>
2475 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz")
2476 (match_operand:VD_BHSI 1 "register_operand" "w")))]
2477 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2478 "mov\\t%0.8b, %1.8b"
2479 [(set_attr "type" "neon_move<q>")]
2480 )
2481
2482 (define_expand "aarch64_combine<mode>"
2483 [(match_operand:<VDBL> 0 "register_operand")
2484 (match_operand:VDC 1 "register_operand")
2485 (match_operand:VDC 2 "register_operand")]
2486 "TARGET_SIMD"
2487 {
2488 rtx op1, op2;
2489 if (BYTES_BIG_ENDIAN)
2490 {
2491 op1 = operands[2];
2492 op2 = operands[1];
2493 }
2494 else
2495 {
2496 op1 = operands[1];
2497 op2 = operands[2];
2498 }
2499 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2500 DONE;
2501 }
2502 )
2503
2504 (define_insn_and_split "aarch64_combine_internal<mode>"
2505 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2506 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2507 (match_operand:VDC 2 "register_operand" "w")))]
2508 "TARGET_SIMD"
2509 "#"
2510 "&& reload_completed"
2511 [(const_int 0)]
2512 {
2513 if (BYTES_BIG_ENDIAN)
2514 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2515 else
2516 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2517 DONE;
2518 }
2519 [(set_attr "type" "multiple")]
2520 )
2521
2522 (define_expand "aarch64_simd_combine<mode>"
2523 [(match_operand:<VDBL> 0 "register_operand")
2524 (match_operand:VDC 1 "register_operand")
2525 (match_operand:VDC 2 "register_operand")]
2526 "TARGET_SIMD"
2527 {
2528 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2529 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2530 DONE;
2531 }
2532 [(set_attr "type" "multiple")]
2533 )
2534
2535 ;; <su><addsub>l<q>.
2536
2537 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2538 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2539 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2540 (match_operand:VQW 1 "register_operand" "w")
2541 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2542 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2543 (match_operand:VQW 2 "register_operand" "w")
2544 (match_dup 3)))))]
2545 "TARGET_SIMD"
2546 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2547 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2548 )
2549
2550 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2551 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2552 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2553 (match_operand:VQW 1 "register_operand" "w")
2554 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2555 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2556 (match_operand:VQW 2 "register_operand" "w")
2557 (match_dup 3)))))]
2558 "TARGET_SIMD"
2559 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2560 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2561 )
2562
2563
2564 (define_expand "aarch64_saddl2<mode>"
2565 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2566 (match_operand:VQW 1 "register_operand" "w")
2567 (match_operand:VQW 2 "register_operand" "w")]
2568 "TARGET_SIMD"
2569 {
2570 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2571 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2572 operands[2], p));
2573 DONE;
2574 })
2575
2576 (define_expand "aarch64_uaddl2<mode>"
2577 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2578 (match_operand:VQW 1 "register_operand" "w")
2579 (match_operand:VQW 2 "register_operand" "w")]
2580 "TARGET_SIMD"
2581 {
2582 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2583 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2584 operands[2], p));
2585 DONE;
2586 })
2587
2588 (define_expand "aarch64_ssubl2<mode>"
2589 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2590 (match_operand:VQW 1 "register_operand" "w")
2591 (match_operand:VQW 2 "register_operand" "w")]
2592 "TARGET_SIMD"
2593 {
2594 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2595 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2596 operands[2], p));
2597 DONE;
2598 })
2599
2600 (define_expand "aarch64_usubl2<mode>"
2601 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2602 (match_operand:VQW 1 "register_operand" "w")
2603 (match_operand:VQW 2 "register_operand" "w")]
2604 "TARGET_SIMD"
2605 {
2606 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2607 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2608 operands[2], p));
2609 DONE;
2610 })
2611
2612 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2613 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2614 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2615 (match_operand:VD_BHSI 1 "register_operand" "w"))
2616 (ANY_EXTEND:<VWIDE>
2617 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2618 "TARGET_SIMD"
2619 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2620 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2621 )
2622
2623 ;; <su><addsub>w<q>.
2624
2625 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2626 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2627 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2628 (ANY_EXTEND:<VWIDE>
2629 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2630 "TARGET_SIMD"
2631 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2632 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2633 )
2634
2635 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2636 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2637 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2638 (ANY_EXTEND:<VWIDE>
2639 (vec_select:<VHALF>
2640 (match_operand:VQW 2 "register_operand" "w")
2641 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2642 "TARGET_SIMD"
2643 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2644 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2645 )
2646
2647 (define_expand "aarch64_saddw2<mode>"
2648 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2649 (match_operand:<VWIDE> 1 "register_operand" "w")
2650 (match_operand:VQW 2 "register_operand" "w")]
2651 "TARGET_SIMD"
2652 {
2653 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2654 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2655 operands[2], p));
2656 DONE;
2657 })
2658
2659 (define_expand "aarch64_uaddw2<mode>"
2660 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2661 (match_operand:<VWIDE> 1 "register_operand" "w")
2662 (match_operand:VQW 2 "register_operand" "w")]
2663 "TARGET_SIMD"
2664 {
2665 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2666 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2667 operands[2], p));
2668 DONE;
2669 })
2670
2671
2672 (define_expand "aarch64_ssubw2<mode>"
2673 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2674 (match_operand:<VWIDE> 1 "register_operand" "w")
2675 (match_operand:VQW 2 "register_operand" "w")]
2676 "TARGET_SIMD"
2677 {
2678 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2679 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2680 operands[2], p));
2681 DONE;
2682 })
2683
2684 (define_expand "aarch64_usubw2<mode>"
2685 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2686 (match_operand:<VWIDE> 1 "register_operand" "w")
2687 (match_operand:VQW 2 "register_operand" "w")]
2688 "TARGET_SIMD"
2689 {
2690 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2691 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2692 operands[2], p));
2693 DONE;
2694 })
2695
2696 ;; <su><r>h<addsub>.
2697
2698 (define_insn "aarch64_<sur>h<addsub><mode>"
2699 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2700 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
2701 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
2702 HADDSUB))]
2703 "TARGET_SIMD"
2704 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2705 [(set_attr "type" "neon_<addsub>_halve<q>")]
2706 )
2707
2708 ;; <r><addsub>hn<q>.
2709
2710 (define_insn "aarch64_<sur><addsub>hn<mode>"
2711 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2712 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2713 (match_operand:VQN 2 "register_operand" "w")]
2714 ADDSUBHN))]
2715 "TARGET_SIMD"
2716 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2717 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2718 )
2719
2720 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2721 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2722 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2723 (match_operand:VQN 2 "register_operand" "w")
2724 (match_operand:VQN 3 "register_operand" "w")]
2725 ADDSUBHN2))]
2726 "TARGET_SIMD"
2727 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2728 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2729 )
2730
2731 ;; pmul.
2732
2733 (define_insn "aarch64_pmul<mode>"
2734 [(set (match_operand:VB 0 "register_operand" "=w")
2735 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2736 (match_operand:VB 2 "register_operand" "w")]
2737 UNSPEC_PMUL))]
2738 "TARGET_SIMD"
2739 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2740 [(set_attr "type" "neon_mul_<Vetype><q>")]
2741 )
2742
2743 ;; <su>q<addsub>
2744
2745 (define_insn "aarch64_<su_optab><optab><mode>"
2746 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2747 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2748 (match_operand:VSDQ_I 2 "register_operand" "w")))]
2749 "TARGET_SIMD"
2750 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2751 [(set_attr "type" "neon_<optab><q>")]
2752 )
2753
2754 ;; suqadd and usqadd
2755
2756 (define_insn "aarch64_<sur>qadd<mode>"
2757 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2758 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2759 (match_operand:VSDQ_I 2 "register_operand" "w")]
2760 USSUQADD))]
2761 "TARGET_SIMD"
2762 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2763 [(set_attr "type" "neon_qadd<q>")]
2764 )
2765
2766 ;; sqmovun
2767
2768 (define_insn "aarch64_sqmovun<mode>"
2769 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2770 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2771 UNSPEC_SQXTUN))]
2772 "TARGET_SIMD"
2773 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2774 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2775 )
2776
2777 ;; sqmovn and uqmovn
2778
2779 (define_insn "aarch64_<sur>qmovn<mode>"
2780 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2781 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2782 SUQMOVN))]
2783 "TARGET_SIMD"
2784 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2785 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2786 )
2787
2788 ;; <su>q<absneg>
2789
2790 (define_insn "aarch64_s<optab><mode>"
2791 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2792 (UNQOPS:VSDQ_I
2793 (match_operand:VSDQ_I 1 "register_operand" "w")))]
2794 "TARGET_SIMD"
2795 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2796 [(set_attr "type" "neon_<optab><q>")]
2797 )
2798
2799 ;; sq<r>dmulh.
2800
2801 (define_insn "aarch64_sq<r>dmulh<mode>"
2802 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2803 (unspec:VSDQ_HSI
2804 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2805 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2806 VQDMULH))]
2807 "TARGET_SIMD"
2808 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2809 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
2810 )
2811
2812 ;; sq<r>dmulh_lane
2813
2814 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2815 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2816 (unspec:VDQHS
2817 [(match_operand:VDQHS 1 "register_operand" "w")
2818 (vec_select:<VEL>
2819 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2820 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2821 VQDMULH))]
2822 "TARGET_SIMD"
2823 "*
2824 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2825 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2826 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2827 )
2828
2829 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2830 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2831 (unspec:VDQHS
2832 [(match_operand:VDQHS 1 "register_operand" "w")
2833 (vec_select:<VEL>
2834 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2835 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2836 VQDMULH))]
2837 "TARGET_SIMD"
2838 "*
2839 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2840 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2841 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2842 )
2843
2844 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2845 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2846 (unspec:SD_HSI
2847 [(match_operand:SD_HSI 1 "register_operand" "w")
2848 (vec_select:<VEL>
2849 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2850 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2851 VQDMULH))]
2852 "TARGET_SIMD"
2853 "*
2854 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2855 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2856 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2857 )
2858
2859 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2860 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2861 (unspec:SD_HSI
2862 [(match_operand:SD_HSI 1 "register_operand" "w")
2863 (vec_select:<VEL>
2864 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2865 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2866 VQDMULH))]
2867 "TARGET_SIMD"
2868 "*
2869 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2870 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2871 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2872 )
2873
2874 ;; vqdml[sa]l
2875
2876 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2877 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2878 (SBINQOPS:<VWIDE>
2879 (match_operand:<VWIDE> 1 "register_operand" "0")
2880 (ss_ashift:<VWIDE>
2881 (mult:<VWIDE>
2882 (sign_extend:<VWIDE>
2883 (match_operand:VSD_HSI 2 "register_operand" "w"))
2884 (sign_extend:<VWIDE>
2885 (match_operand:VSD_HSI 3 "register_operand" "w")))
2886 (const_int 1))))]
2887 "TARGET_SIMD"
2888 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2889 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
2890 )
2891
2892 ;; vqdml[sa]l_lane
2893
2894 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
2895 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2896 (SBINQOPS:<VWIDE>
2897 (match_operand:<VWIDE> 1 "register_operand" "0")
2898 (ss_ashift:<VWIDE>
2899 (mult:<VWIDE>
2900 (sign_extend:<VWIDE>
2901 (match_operand:VD_HSI 2 "register_operand" "w"))
2902 (sign_extend:<VWIDE>
2903 (vec_duplicate:VD_HSI
2904 (vec_select:<VEL>
2905 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2906 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2907 ))
2908 (const_int 1))))]
2909 "TARGET_SIMD"
2910 {
2911 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2912 return
2913 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2914 }
2915 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2916 )
2917
2918 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
2919 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2920 (SBINQOPS:<VWIDE>
2921 (match_operand:<VWIDE> 1 "register_operand" "0")
2922 (ss_ashift:<VWIDE>
2923 (mult:<VWIDE>
2924 (sign_extend:<VWIDE>
2925 (match_operand:VD_HSI 2 "register_operand" "w"))
2926 (sign_extend:<VWIDE>
2927 (vec_duplicate:VD_HSI
2928 (vec_select:<VEL>
2929 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
2930 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2931 ))
2932 (const_int 1))))]
2933 "TARGET_SIMD"
2934 {
2935 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2936 return
2937 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2938 }
2939 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2940 )
2941
2942 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
2943 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2944 (SBINQOPS:<VWIDE>
2945 (match_operand:<VWIDE> 1 "register_operand" "0")
2946 (ss_ashift:<VWIDE>
2947 (mult:<VWIDE>
2948 (sign_extend:<VWIDE>
2949 (match_operand:SD_HSI 2 "register_operand" "w"))
2950 (sign_extend:<VWIDE>
2951 (vec_select:<VEL>
2952 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2953 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2954 )
2955 (const_int 1))))]
2956 "TARGET_SIMD"
2957 {
2958 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2959 return
2960 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2961 }
2962 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2963 )
2964
2965 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
2966 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2967 (SBINQOPS:<VWIDE>
2968 (match_operand:<VWIDE> 1 "register_operand" "0")
2969 (ss_ashift:<VWIDE>
2970 (mult:<VWIDE>
2971 (sign_extend:<VWIDE>
2972 (match_operand:SD_HSI 2 "register_operand" "w"))
2973 (sign_extend:<VWIDE>
2974 (vec_select:<VEL>
2975 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
2976 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2977 )
2978 (const_int 1))))]
2979 "TARGET_SIMD"
2980 {
2981 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2982 return
2983 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2984 }
2985 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2986 )
2987
2988 ;; vqdml[sa]l_n
2989
2990 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2991 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2992 (SBINQOPS:<VWIDE>
2993 (match_operand:<VWIDE> 1 "register_operand" "0")
2994 (ss_ashift:<VWIDE>
2995 (mult:<VWIDE>
2996 (sign_extend:<VWIDE>
2997 (match_operand:VD_HSI 2 "register_operand" "w"))
2998 (sign_extend:<VWIDE>
2999 (vec_duplicate:VD_HSI
3000 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3001 (const_int 1))))]
3002 "TARGET_SIMD"
3003 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3004 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3005 )
3006
3007 ;; sqdml[as]l2
3008
3009 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3010 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3011 (SBINQOPS:<VWIDE>
3012 (match_operand:<VWIDE> 1 "register_operand" "0")
3013 (ss_ashift:<VWIDE>
3014 (mult:<VWIDE>
3015 (sign_extend:<VWIDE>
3016 (vec_select:<VHALF>
3017 (match_operand:VQ_HSI 2 "register_operand" "w")
3018 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3019 (sign_extend:<VWIDE>
3020 (vec_select:<VHALF>
3021 (match_operand:VQ_HSI 3 "register_operand" "w")
3022 (match_dup 4))))
3023 (const_int 1))))]
3024 "TARGET_SIMD"
3025 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3026 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3027 )
3028
3029 (define_expand "aarch64_sqdmlal2<mode>"
3030 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3031 (match_operand:<VWIDE> 1 "register_operand" "w")
3032 (match_operand:VQ_HSI 2 "register_operand" "w")
3033 (match_operand:VQ_HSI 3 "register_operand" "w")]
3034 "TARGET_SIMD"
3035 {
3036 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3037 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3038 operands[2], operands[3], p));
3039 DONE;
3040 })
3041
3042 (define_expand "aarch64_sqdmlsl2<mode>"
3043 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3044 (match_operand:<VWIDE> 1 "register_operand" "w")
3045 (match_operand:VQ_HSI 2 "register_operand" "w")
3046 (match_operand:VQ_HSI 3 "register_operand" "w")]
3047 "TARGET_SIMD"
3048 {
3049 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3050 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3051 operands[2], operands[3], p));
3052 DONE;
3053 })
3054
3055 ;; vqdml[sa]l2_lane
3056
3057 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3058 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3059 (SBINQOPS:<VWIDE>
3060 (match_operand:<VWIDE> 1 "register_operand" "0")
3061 (ss_ashift:<VWIDE>
3062 (mult:<VWIDE>
3063 (sign_extend:<VWIDE>
3064 (vec_select:<VHALF>
3065 (match_operand:VQ_HSI 2 "register_operand" "w")
3066 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3067 (sign_extend:<VWIDE>
3068 (vec_duplicate:<VHALF>
3069 (vec_select:<VEL>
3070 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3071 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3072 ))))
3073 (const_int 1))))]
3074 "TARGET_SIMD"
3075 {
3076 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3077 return
3078 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3079 }
3080 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3081 )
3082
3083 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3084 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3085 (SBINQOPS:<VWIDE>
3086 (match_operand:<VWIDE> 1 "register_operand" "0")
3087 (ss_ashift:<VWIDE>
3088 (mult:<VWIDE>
3089 (sign_extend:<VWIDE>
3090 (vec_select:<VHALF>
3091 (match_operand:VQ_HSI 2 "register_operand" "w")
3092 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3093 (sign_extend:<VWIDE>
3094 (vec_duplicate:<VHALF>
3095 (vec_select:<VEL>
3096 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3097 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3098 ))))
3099 (const_int 1))))]
3100 "TARGET_SIMD"
3101 {
3102 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3103 return
3104 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3105 }
3106 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3107 )
3108
3109 (define_expand "aarch64_sqdmlal2_lane<mode>"
3110 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3111 (match_operand:<VWIDE> 1 "register_operand" "w")
3112 (match_operand:VQ_HSI 2 "register_operand" "w")
3113 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3114 (match_operand:SI 4 "immediate_operand" "i")]
3115 "TARGET_SIMD"
3116 {
3117 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3118 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3119 operands[2], operands[3],
3120 operands[4], p));
3121 DONE;
3122 })
3123
3124 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3125 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3126 (match_operand:<VWIDE> 1 "register_operand" "w")
3127 (match_operand:VQ_HSI 2 "register_operand" "w")
3128 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3129 (match_operand:SI 4 "immediate_operand" "i")]
3130 "TARGET_SIMD"
3131 {
3132 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3133 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3134 operands[2], operands[3],
3135 operands[4], p));
3136 DONE;
3137 })
3138
3139 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3140 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3141 (match_operand:<VWIDE> 1 "register_operand" "w")
3142 (match_operand:VQ_HSI 2 "register_operand" "w")
3143 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3144 (match_operand:SI 4 "immediate_operand" "i")]
3145 "TARGET_SIMD"
3146 {
3147 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3148 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3149 operands[2], operands[3],
3150 operands[4], p));
3151 DONE;
3152 })
3153
3154 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3155 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3156 (match_operand:<VWIDE> 1 "register_operand" "w")
3157 (match_operand:VQ_HSI 2 "register_operand" "w")
3158 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3159 (match_operand:SI 4 "immediate_operand" "i")]
3160 "TARGET_SIMD"
3161 {
3162 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3163 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3164 operands[2], operands[3],
3165 operands[4], p));
3166 DONE;
3167 })
3168
3169 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3170 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3171 (SBINQOPS:<VWIDE>
3172 (match_operand:<VWIDE> 1 "register_operand" "0")
3173 (ss_ashift:<VWIDE>
3174 (mult:<VWIDE>
3175 (sign_extend:<VWIDE>
3176 (vec_select:<VHALF>
3177 (match_operand:VQ_HSI 2 "register_operand" "w")
3178 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3179 (sign_extend:<VWIDE>
3180 (vec_duplicate:<VHALF>
3181 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3182 (const_int 1))))]
3183 "TARGET_SIMD"
3184 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3185 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3186 )
3187
3188 (define_expand "aarch64_sqdmlal2_n<mode>"
3189 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3190 (match_operand:<VWIDE> 1 "register_operand" "w")
3191 (match_operand:VQ_HSI 2 "register_operand" "w")
3192 (match_operand:<VEL> 3 "register_operand" "w")]
3193 "TARGET_SIMD"
3194 {
3195 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3196 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3197 operands[2], operands[3],
3198 p));
3199 DONE;
3200 })
3201
3202 (define_expand "aarch64_sqdmlsl2_n<mode>"
3203 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3204 (match_operand:<VWIDE> 1 "register_operand" "w")
3205 (match_operand:VQ_HSI 2 "register_operand" "w")
3206 (match_operand:<VEL> 3 "register_operand" "w")]
3207 "TARGET_SIMD"
3208 {
3209 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3210 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3211 operands[2], operands[3],
3212 p));
3213 DONE;
3214 })
3215
3216 ;; vqdmull
3217
3218 (define_insn "aarch64_sqdmull<mode>"
3219 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3220 (ss_ashift:<VWIDE>
3221 (mult:<VWIDE>
3222 (sign_extend:<VWIDE>
3223 (match_operand:VSD_HSI 1 "register_operand" "w"))
3224 (sign_extend:<VWIDE>
3225 (match_operand:VSD_HSI 2 "register_operand" "w")))
3226 (const_int 1)))]
3227 "TARGET_SIMD"
3228 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3229 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3230 )
3231
3232 ;; vqdmull_lane
3233
3234 (define_insn "aarch64_sqdmull_lane<mode>"
3235 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3236 (ss_ashift:<VWIDE>
3237 (mult:<VWIDE>
3238 (sign_extend:<VWIDE>
3239 (match_operand:VD_HSI 1 "register_operand" "w"))
3240 (sign_extend:<VWIDE>
3241 (vec_duplicate:VD_HSI
3242 (vec_select:<VEL>
3243 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3244 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3245 ))
3246 (const_int 1)))]
3247 "TARGET_SIMD"
3248 {
3249 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3250 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3251 }
3252 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3253 )
3254
3255 (define_insn "aarch64_sqdmull_laneq<mode>"
3256 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3257 (ss_ashift:<VWIDE>
3258 (mult:<VWIDE>
3259 (sign_extend:<VWIDE>
3260 (match_operand:VD_HSI 1 "register_operand" "w"))
3261 (sign_extend:<VWIDE>
3262 (vec_duplicate:VD_HSI
3263 (vec_select:<VEL>
3264 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3265 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3266 ))
3267 (const_int 1)))]
3268 "TARGET_SIMD"
3269 {
3270 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3271 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3272 }
3273 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3274 )
3275
3276 (define_insn "aarch64_sqdmull_lane<mode>"
3277 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3278 (ss_ashift:<VWIDE>
3279 (mult:<VWIDE>
3280 (sign_extend:<VWIDE>
3281 (match_operand:SD_HSI 1 "register_operand" "w"))
3282 (sign_extend:<VWIDE>
3283 (vec_select:<VEL>
3284 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3285 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3286 ))
3287 (const_int 1)))]
3288 "TARGET_SIMD"
3289 {
3290 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3291 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3292 }
3293 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3294 )
3295
3296 (define_insn "aarch64_sqdmull_laneq<mode>"
3297 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3298 (ss_ashift:<VWIDE>
3299 (mult:<VWIDE>
3300 (sign_extend:<VWIDE>
3301 (match_operand:SD_HSI 1 "register_operand" "w"))
3302 (sign_extend:<VWIDE>
3303 (vec_select:<VEL>
3304 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3305 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3306 ))
3307 (const_int 1)))]
3308 "TARGET_SIMD"
3309 {
3310 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3311 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3312 }
3313 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3314 )
3315
3316 ;; vqdmull_n
3317
3318 (define_insn "aarch64_sqdmull_n<mode>"
3319 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3320 (ss_ashift:<VWIDE>
3321 (mult:<VWIDE>
3322 (sign_extend:<VWIDE>
3323 (match_operand:VD_HSI 1 "register_operand" "w"))
3324 (sign_extend:<VWIDE>
3325 (vec_duplicate:VD_HSI
3326 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3327 )
3328 (const_int 1)))]
3329 "TARGET_SIMD"
3330 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3331 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3332 )
3333
3334 ;; vqdmull2
3335
3336
3337
3338 (define_insn "aarch64_sqdmull2<mode>_internal"
3339 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3340 (ss_ashift:<VWIDE>
3341 (mult:<VWIDE>
3342 (sign_extend:<VWIDE>
3343 (vec_select:<VHALF>
3344 (match_operand:VQ_HSI 1 "register_operand" "w")
3345 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3346 (sign_extend:<VWIDE>
3347 (vec_select:<VHALF>
3348 (match_operand:VQ_HSI 2 "register_operand" "w")
3349 (match_dup 3)))
3350 )
3351 (const_int 1)))]
3352 "TARGET_SIMD"
3353 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3354 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3355 )
3356
3357 (define_expand "aarch64_sqdmull2<mode>"
3358 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3359 (match_operand:VQ_HSI 1 "register_operand" "w")
3360 (match_operand:VQ_HSI 2 "register_operand" "w")]
3361 "TARGET_SIMD"
3362 {
3363 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3364 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3365 operands[2], p));
3366 DONE;
3367 })
3368
3369 ;; vqdmull2_lane
3370
3371 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3372 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3373 (ss_ashift:<VWIDE>
3374 (mult:<VWIDE>
3375 (sign_extend:<VWIDE>
3376 (vec_select:<VHALF>
3377 (match_operand:VQ_HSI 1 "register_operand" "w")
3378 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3379 (sign_extend:<VWIDE>
3380 (vec_duplicate:<VHALF>
3381 (vec_select:<VEL>
3382 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3383 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3384 ))
3385 (const_int 1)))]
3386 "TARGET_SIMD"
3387 {
3388 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3389 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3390 }
3391 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3392 )
3393
3394 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3395 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3396 (ss_ashift:<VWIDE>
3397 (mult:<VWIDE>
3398 (sign_extend:<VWIDE>
3399 (vec_select:<VHALF>
3400 (match_operand:VQ_HSI 1 "register_operand" "w")
3401 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3402 (sign_extend:<VWIDE>
3403 (vec_duplicate:<VHALF>
3404 (vec_select:<VEL>
3405 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3406 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3407 ))
3408 (const_int 1)))]
3409 "TARGET_SIMD"
3410 {
3411 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3412 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3413 }
3414 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3415 )
3416
3417 (define_expand "aarch64_sqdmull2_lane<mode>"
3418 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3419 (match_operand:VQ_HSI 1 "register_operand" "w")
3420 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3421 (match_operand:SI 3 "immediate_operand" "i")]
3422 "TARGET_SIMD"
3423 {
3424 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3425 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3426 operands[2], operands[3],
3427 p));
3428 DONE;
3429 })
3430
3431 (define_expand "aarch64_sqdmull2_laneq<mode>"
3432 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3433 (match_operand:VQ_HSI 1 "register_operand" "w")
3434 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3435 (match_operand:SI 3 "immediate_operand" "i")]
3436 "TARGET_SIMD"
3437 {
3438 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3439 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3440 operands[2], operands[3],
3441 p));
3442 DONE;
3443 })
3444
3445 ;; vqdmull2_n
3446
3447 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3448 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3449 (ss_ashift:<VWIDE>
3450 (mult:<VWIDE>
3451 (sign_extend:<VWIDE>
3452 (vec_select:<VHALF>
3453 (match_operand:VQ_HSI 1 "register_operand" "w")
3454 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3455 (sign_extend:<VWIDE>
3456 (vec_duplicate:<VHALF>
3457 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3458 )
3459 (const_int 1)))]
3460 "TARGET_SIMD"
3461 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3462 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3463 )
3464
3465 (define_expand "aarch64_sqdmull2_n<mode>"
3466 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3467 (match_operand:VQ_HSI 1 "register_operand" "w")
3468 (match_operand:<VEL> 2 "register_operand" "w")]
3469 "TARGET_SIMD"
3470 {
3471 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3472 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3473 operands[2], p));
3474 DONE;
3475 })
3476
3477 ;; vshl
3478
3479 (define_insn "aarch64_<sur>shl<mode>"
3480 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3481 (unspec:VSDQ_I_DI
3482 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3483 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3484 VSHL))]
3485 "TARGET_SIMD"
3486 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3487 [(set_attr "type" "neon_shift_reg<q>")]
3488 )
3489
3490
3491 ;; vqshl
3492
3493 (define_insn "aarch64_<sur>q<r>shl<mode>"
3494 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3495 (unspec:VSDQ_I
3496 [(match_operand:VSDQ_I 1 "register_operand" "w")
3497 (match_operand:VSDQ_I 2 "register_operand" "w")]
3498 VQSHL))]
3499 "TARGET_SIMD"
3500 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3501 [(set_attr "type" "neon_sat_shift_reg<q>")]
3502 )
3503
3504 ;; vshll_n
3505
3506 (define_insn "aarch64_<sur>shll_n<mode>"
3507 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3508 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
3509 (match_operand:SI 2
3510 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
3511 VSHLL))]
3512 "TARGET_SIMD"
3513 "*
3514 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3515 if (INTVAL (operands[2]) == bit_width)
3516 {
3517 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3518 }
3519 else {
3520 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3521 }"
3522 [(set_attr "type" "neon_shift_imm_long")]
3523 )
3524
3525 ;; vshll_high_n
3526
3527 (define_insn "aarch64_<sur>shll2_n<mode>"
3528 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3529 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3530 (match_operand:SI 2 "immediate_operand" "i")]
3531 VSHLL))]
3532 "TARGET_SIMD"
3533 "*
3534 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3535 if (INTVAL (operands[2]) == bit_width)
3536 {
3537 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3538 }
3539 else {
3540 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3541 }"
3542 [(set_attr "type" "neon_shift_imm_long")]
3543 )
3544
3545 ;; vrshr_n
3546
3547 (define_insn "aarch64_<sur>shr_n<mode>"
3548 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3549 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3550 (match_operand:SI 2
3551 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3552 VRSHR_N))]
3553 "TARGET_SIMD"
3554 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3555 [(set_attr "type" "neon_sat_shift_imm<q>")]
3556 )
3557
3558 ;; v(r)sra_n
3559
3560 (define_insn "aarch64_<sur>sra_n<mode>"
3561 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3562 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3563 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3564 (match_operand:SI 3
3565 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3566 VSRA))]
3567 "TARGET_SIMD"
3568 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3569 [(set_attr "type" "neon_shift_acc<q>")]
3570 )
3571
3572 ;; vs<lr>i_n
3573
3574 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3575 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3576 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3577 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3578 (match_operand:SI 3
3579 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
3580 VSLRI))]
3581 "TARGET_SIMD"
3582 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3583 [(set_attr "type" "neon_shift_imm<q>")]
3584 )
3585
3586 ;; vqshl(u)
3587
3588 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3589 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3590 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3591 (match_operand:SI 2
3592 "aarch64_simd_shift_imm_<ve_mode>" "i")]
3593 VQSHL_N))]
3594 "TARGET_SIMD"
3595 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3596 [(set_attr "type" "neon_sat_shift_imm<q>")]
3597 )
3598
3599
3600 ;; vq(r)shr(u)n_n
3601
3602 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3603 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3604 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3605 (match_operand:SI 2
3606 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3607 VQSHRN_N))]
3608 "TARGET_SIMD"
3609 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
3610 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3611 )
3612
3613
3614 ;; cm(eq|ge|gt|lt|le)
3615 ;; Note, we have constraints for Dz and Z as different expanders
3616 ;; have different ideas of what should be passed to this pattern.
3617
3618 (define_insn "aarch64_cm<optab><mode>"
3619 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3620 (neg:<V_cmp_result>
3621 (COMPARISONS:<V_cmp_result>
3622 (match_operand:VDQ_I 1 "register_operand" "w,w")
3623 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3624 )))]
3625 "TARGET_SIMD"
3626 "@
3627 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3628 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3629 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
3630 )
3631
3632 (define_insn_and_split "aarch64_cm<optab>di"
3633 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3634 (neg:DI
3635 (COMPARISONS:DI
3636 (match_operand:DI 1 "register_operand" "w,w,r")
3637 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3638 )))
3639 (clobber (reg:CC CC_REGNUM))]
3640 "TARGET_SIMD"
3641 "#"
3642 "reload_completed"
3643 [(set (match_operand:DI 0 "register_operand")
3644 (neg:DI
3645 (COMPARISONS:DI
3646 (match_operand:DI 1 "register_operand")
3647 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3648 )))]
3649 {
3650 /* If we are in the general purpose register file,
3651 we split to a sequence of comparison and store. */
3652 if (GP_REGNUM_P (REGNO (operands[0]))
3653 && GP_REGNUM_P (REGNO (operands[1])))
3654 {
3655 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3656 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3657 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3658 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3659 DONE;
3660 }
3661 /* Otherwise, we expand to a similar pattern which does not
3662 clobber CC_REGNUM. */
3663 }
3664 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
3665 )
3666
3667 (define_insn "*aarch64_cm<optab>di"
3668 [(set (match_operand:DI 0 "register_operand" "=w,w")
3669 (neg:DI
3670 (COMPARISONS:DI
3671 (match_operand:DI 1 "register_operand" "w,w")
3672 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3673 )))]
3674 "TARGET_SIMD && reload_completed"
3675 "@
3676 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3677 cm<optab>\t%d0, %d1, #0"
3678 [(set_attr "type" "neon_compare, neon_compare_zero")]
3679 )
3680
3681 ;; cm(hs|hi)
3682
3683 (define_insn "aarch64_cm<optab><mode>"
3684 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3685 (neg:<V_cmp_result>
3686 (UCOMPARISONS:<V_cmp_result>
3687 (match_operand:VDQ_I 1 "register_operand" "w")
3688 (match_operand:VDQ_I 2 "register_operand" "w")
3689 )))]
3690 "TARGET_SIMD"
3691 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3692 [(set_attr "type" "neon_compare<q>")]
3693 )
3694
3695 (define_insn_and_split "aarch64_cm<optab>di"
3696 [(set (match_operand:DI 0 "register_operand" "=w,r")
3697 (neg:DI
3698 (UCOMPARISONS:DI
3699 (match_operand:DI 1 "register_operand" "w,r")
3700 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3701 )))
3702 (clobber (reg:CC CC_REGNUM))]
3703 "TARGET_SIMD"
3704 "#"
3705 "reload_completed"
3706 [(set (match_operand:DI 0 "register_operand")
3707 (neg:DI
3708 (UCOMPARISONS:DI
3709 (match_operand:DI 1 "register_operand")
3710 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3711 )))]
3712 {
3713 /* If we are in the general purpose register file,
3714 we split to a sequence of comparison and store. */
3715 if (GP_REGNUM_P (REGNO (operands[0]))
3716 && GP_REGNUM_P (REGNO (operands[1])))
3717 {
3718 machine_mode mode = CCmode;
3719 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3720 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3721 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3722 DONE;
3723 }
3724 /* Otherwise, we expand to a similar pattern which does not
3725 clobber CC_REGNUM. */
3726 }
3727 [(set_attr "type" "neon_compare,multiple")]
3728 )
3729
3730 (define_insn "*aarch64_cm<optab>di"
3731 [(set (match_operand:DI 0 "register_operand" "=w")
3732 (neg:DI
3733 (UCOMPARISONS:DI
3734 (match_operand:DI 1 "register_operand" "w")
3735 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
3736 )))]
3737 "TARGET_SIMD && reload_completed"
3738 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
3739 [(set_attr "type" "neon_compare")]
3740 )
3741
3742 ;; cmtst
3743
3744 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
3745 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
3746 ;; not (neg (eq (and x y) 0))
3747 ;; which is rewritten by simplify_rtx as
3748 ;; plus (eq (and x y) 0) -1.
3749
3750 (define_insn "aarch64_cmtst<mode>"
3751 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3752 (plus:<V_cmp_result>
3753 (eq:<V_cmp_result>
3754 (and:VDQ_I
3755 (match_operand:VDQ_I 1 "register_operand" "w")
3756 (match_operand:VDQ_I 2 "register_operand" "w"))
3757 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
3758 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
3759 ]
3760 "TARGET_SIMD"
3761 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3762 [(set_attr "type" "neon_tst<q>")]
3763 )
3764
3765 (define_insn_and_split "aarch64_cmtstdi"
3766 [(set (match_operand:DI 0 "register_operand" "=w,r")
3767 (neg:DI
3768 (ne:DI
3769 (and:DI
3770 (match_operand:DI 1 "register_operand" "w,r")
3771 (match_operand:DI 2 "register_operand" "w,r"))
3772 (const_int 0))))
3773 (clobber (reg:CC CC_REGNUM))]
3774 "TARGET_SIMD"
3775 "#"
3776 "reload_completed"
3777 [(set (match_operand:DI 0 "register_operand")
3778 (neg:DI
3779 (ne:DI
3780 (and:DI
3781 (match_operand:DI 1 "register_operand")
3782 (match_operand:DI 2 "register_operand"))
3783 (const_int 0))))]
3784 {
3785 /* If we are in the general purpose register file,
3786 we split to a sequence of comparison and store. */
3787 if (GP_REGNUM_P (REGNO (operands[0]))
3788 && GP_REGNUM_P (REGNO (operands[1])))
3789 {
3790 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3791 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3792 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3793 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3794 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3795 DONE;
3796 }
3797 /* Otherwise, we expand to a similar pattern which does not
3798 clobber CC_REGNUM. */
3799 }
3800 [(set_attr "type" "neon_tst,multiple")]
3801 )
3802
3803 (define_insn "*aarch64_cmtstdi"
3804 [(set (match_operand:DI 0 "register_operand" "=w")
3805 (neg:DI
3806 (ne:DI
3807 (and:DI
3808 (match_operand:DI 1 "register_operand" "w")
3809 (match_operand:DI 2 "register_operand" "w"))
3810 (const_int 0))))]
3811 "TARGET_SIMD"
3812 "cmtst\t%d0, %d1, %d2"
3813 [(set_attr "type" "neon_tst")]
3814 )
3815
3816 ;; fcm(eq|ge|gt|le|lt)
3817
3818 (define_insn "aarch64_cm<optab><mode>"
3819 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3820 (neg:<V_cmp_result>
3821 (COMPARISONS:<V_cmp_result>
3822 (match_operand:VALLF 1 "register_operand" "w,w")
3823 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3824 )))]
3825 "TARGET_SIMD"
3826 "@
3827 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3828 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3829 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
3830 )
3831
3832 ;; fac(ge|gt)
3833 ;; Note we can also handle what would be fac(le|lt) by
3834 ;; generating fac(ge|gt).
3835
3836 (define_insn "*aarch64_fac<optab><mode>"
3837 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3838 (neg:<V_cmp_result>
3839 (FAC_COMPARISONS:<V_cmp_result>
3840 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3841 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3842 )))]
3843 "TARGET_SIMD"
3844 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3845 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
3846 )
3847
3848 ;; addp
3849
3850 (define_insn "aarch64_addp<mode>"
3851 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3852 (unspec:VD_BHSI
3853 [(match_operand:VD_BHSI 1 "register_operand" "w")
3854 (match_operand:VD_BHSI 2 "register_operand" "w")]
3855 UNSPEC_ADDP))]
3856 "TARGET_SIMD"
3857 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3858 [(set_attr "type" "neon_reduc_add<q>")]
3859 )
3860
3861 (define_insn "aarch64_addpdi"
3862 [(set (match_operand:DI 0 "register_operand" "=w")
3863 (unspec:DI
3864 [(match_operand:V2DI 1 "register_operand" "w")]
3865 UNSPEC_ADDP))]
3866 "TARGET_SIMD"
3867 "addp\t%d0, %1.2d"
3868 [(set_attr "type" "neon_reduc_add")]
3869 )
3870
3871 ;; sqrt
3872
3873 (define_insn "sqrt<mode>2"
3874 [(set (match_operand:VDQF 0 "register_operand" "=w")
3875 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3876 "TARGET_SIMD"
3877 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
3878 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
3879 )
3880
3881 ;; Patterns for vector struct loads and stores.
3882
3883 (define_insn "aarch64_simd_ld2<mode>"
3884 [(set (match_operand:OI 0 "register_operand" "=w")
3885 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3886 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3887 UNSPEC_LD2))]
3888 "TARGET_SIMD"
3889 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3890 [(set_attr "type" "neon_load2_2reg<q>")]
3891 )
3892
3893 (define_insn "aarch64_simd_ld2r<mode>"
3894 [(set (match_operand:OI 0 "register_operand" "=w")
3895 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3896 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3897 UNSPEC_LD2_DUP))]
3898 "TARGET_SIMD"
3899 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3900 [(set_attr "type" "neon_load2_all_lanes<q>")]
3901 )
3902
3903 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
3904 [(set (match_operand:OI 0 "register_operand" "=w")
3905 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3906 (match_operand:OI 2 "register_operand" "0")
3907 (match_operand:SI 3 "immediate_operand" "i")
3908 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3909 UNSPEC_LD2_LANE))]
3910 "TARGET_SIMD"
3911 "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1"
3912 [(set_attr "type" "neon_load2_one_lane")]
3913 )
3914
3915 (define_expand "vec_load_lanesoi<mode>"
3916 [(set (match_operand:OI 0 "register_operand" "=w")
3917 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3918 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3919 UNSPEC_LD2))]
3920 "TARGET_SIMD"
3921 {
3922 if (BYTES_BIG_ENDIAN)
3923 {
3924 rtx tmp = gen_reg_rtx (OImode);
3925 rtx mask = aarch64_reverse_mask (<MODE>mode);
3926 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
3927 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
3928 }
3929 else
3930 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
3931 DONE;
3932 })
3933
3934 (define_insn "aarch64_simd_st2<mode>"
3935 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3936 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3937 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3938 UNSPEC_ST2))]
3939 "TARGET_SIMD"
3940 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3941 [(set_attr "type" "neon_store2_2reg<q>")]
3942 )
3943
3944 (define_insn "vec_store_lanesoi_lane<mode>"
3945 [(set (match_operand:<V_TWO_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
3946 (unspec:<V_TWO_ELEM> [(match_operand:OI 1 "register_operand" "w")
3947 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
3948 (match_operand:SI 2 "immediate_operand" "i")]
3949 UNSPEC_ST2_LANE))]
3950 "TARGET_SIMD"
3951 "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0"
3952 [(set_attr "type" "neon_store3_one_lane<q>")]
3953 )
3954
3955 (define_expand "vec_store_lanesoi<mode>"
3956 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3957 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3958 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3959 UNSPEC_ST2))]
3960 "TARGET_SIMD"
3961 {
3962 if (BYTES_BIG_ENDIAN)
3963 {
3964 rtx tmp = gen_reg_rtx (OImode);
3965 rtx mask = aarch64_reverse_mask (<MODE>mode);
3966 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
3967 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
3968 }
3969 else
3970 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
3971 DONE;
3972 })
3973
3974 (define_insn "aarch64_simd_ld3<mode>"
3975 [(set (match_operand:CI 0 "register_operand" "=w")
3976 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3977 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3978 UNSPEC_LD3))]
3979 "TARGET_SIMD"
3980 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3981 [(set_attr "type" "neon_load3_3reg<q>")]
3982 )
3983
3984 (define_insn "aarch64_simd_ld3r<mode>"
3985 [(set (match_operand:CI 0 "register_operand" "=w")
3986 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3987 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3988 UNSPEC_LD3_DUP))]
3989 "TARGET_SIMD"
3990 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3991 [(set_attr "type" "neon_load3_all_lanes<q>")]
3992 )
3993
3994 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
3995 [(set (match_operand:CI 0 "register_operand" "=w")
3996 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3997 (match_operand:CI 2 "register_operand" "0")
3998 (match_operand:SI 3 "immediate_operand" "i")
3999 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4000 UNSPEC_LD3_LANE))]
4001 "TARGET_SIMD"
4002 "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1"
4003 [(set_attr "type" "neon_load3_one_lane")]
4004 )
4005
4006 (define_expand "vec_load_lanesci<mode>"
4007 [(set (match_operand:CI 0 "register_operand" "=w")
4008 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4009 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4010 UNSPEC_LD3))]
4011 "TARGET_SIMD"
4012 {
4013 if (BYTES_BIG_ENDIAN)
4014 {
4015 rtx tmp = gen_reg_rtx (CImode);
4016 rtx mask = aarch64_reverse_mask (<MODE>mode);
4017 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4018 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4019 }
4020 else
4021 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4022 DONE;
4023 })
4024
4025 (define_insn "aarch64_simd_st3<mode>"
4026 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4027 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4028 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4029 UNSPEC_ST3))]
4030 "TARGET_SIMD"
4031 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4032 [(set_attr "type" "neon_store3_3reg<q>")]
4033 )
4034
4035 (define_insn "vec_store_lanesci_lane<mode>"
4036 [(set (match_operand:<V_THREE_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4037 (unspec:<V_THREE_ELEM> [(match_operand:CI 1 "register_operand" "w")
4038 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4039 (match_operand:SI 2 "immediate_operand" "i")]
4040 UNSPEC_ST3_LANE))]
4041 "TARGET_SIMD"
4042 "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0"
4043 [(set_attr "type" "neon_store3_one_lane<q>")]
4044 )
4045
4046 (define_expand "vec_store_lanesci<mode>"
4047 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4048 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4049 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4050 UNSPEC_ST3))]
4051 "TARGET_SIMD"
4052 {
4053 if (BYTES_BIG_ENDIAN)
4054 {
4055 rtx tmp = gen_reg_rtx (CImode);
4056 rtx mask = aarch64_reverse_mask (<MODE>mode);
4057 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4058 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4059 }
4060 else
4061 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4062 DONE;
4063 })
4064
4065 (define_insn "aarch64_simd_ld4<mode>"
4066 [(set (match_operand:XI 0 "register_operand" "=w")
4067 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4068 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4069 UNSPEC_LD4))]
4070 "TARGET_SIMD"
4071 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4072 [(set_attr "type" "neon_load4_4reg<q>")]
4073 )
4074
4075 (define_insn "aarch64_simd_ld4r<mode>"
4076 [(set (match_operand:XI 0 "register_operand" "=w")
4077 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4078 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4079 UNSPEC_LD4_DUP))]
4080 "TARGET_SIMD"
4081 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4082 [(set_attr "type" "neon_load4_all_lanes<q>")]
4083 )
4084
4085 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4086 [(set (match_operand:XI 0 "register_operand" "=w")
4087 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4088 (match_operand:XI 2 "register_operand" "0")
4089 (match_operand:SI 3 "immediate_operand" "i")
4090 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4091 UNSPEC_LD4_LANE))]
4092 "TARGET_SIMD"
4093 "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1"
4094 [(set_attr "type" "neon_load4_one_lane")]
4095 )
4096
4097 (define_expand "vec_load_lanesxi<mode>"
4098 [(set (match_operand:XI 0 "register_operand" "=w")
4099 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4100 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4101 UNSPEC_LD4))]
4102 "TARGET_SIMD"
4103 {
4104 if (BYTES_BIG_ENDIAN)
4105 {
4106 rtx tmp = gen_reg_rtx (XImode);
4107 rtx mask = aarch64_reverse_mask (<MODE>mode);
4108 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4109 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4110 }
4111 else
4112 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4113 DONE;
4114 })
4115
4116 (define_insn "aarch64_simd_st4<mode>"
4117 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4118 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4119 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4120 UNSPEC_ST4))]
4121 "TARGET_SIMD"
4122 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4123 [(set_attr "type" "neon_store4_4reg<q>")]
4124 )
4125
4126 (define_insn "vec_store_lanesxi_lane<mode>"
4127 [(set (match_operand:<V_FOUR_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4128 (unspec:<V_FOUR_ELEM> [(match_operand:XI 1 "register_operand" "w")
4129 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4130 (match_operand:SI 2 "immediate_operand" "i")]
4131 UNSPEC_ST4_LANE))]
4132 "TARGET_SIMD"
4133 "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0"
4134 [(set_attr "type" "neon_store4_one_lane<q>")]
4135 )
4136
4137 (define_expand "vec_store_lanesxi<mode>"
4138 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4139 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4140 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4141 UNSPEC_ST4))]
4142 "TARGET_SIMD"
4143 {
4144 if (BYTES_BIG_ENDIAN)
4145 {
4146 rtx tmp = gen_reg_rtx (XImode);
4147 rtx mask = aarch64_reverse_mask (<MODE>mode);
4148 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4149 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4150 }
4151 else
4152 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4153 DONE;
4154 })
4155
4156 (define_insn_and_split "aarch64_rev_reglist<mode>"
4157 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4158 (unspec:VSTRUCT
4159 [(match_operand:VSTRUCT 1 "register_operand" "w")
4160 (match_operand:V16QI 2 "register_operand" "w")]
4161 UNSPEC_REV_REGLIST))]
4162 "TARGET_SIMD"
4163 "#"
4164 "&& reload_completed"
4165 [(const_int 0)]
4166 {
4167 int i;
4168 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4169 for (i = 0; i < nregs; i++)
4170 {
4171 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4172 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4173 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4174 }
4175 DONE;
4176 }
4177 [(set_attr "type" "neon_tbl1_q")
4178 (set_attr "length" "<insn_count>")]
4179 )
4180
4181 ;; Reload patterns for AdvSIMD register list operands.
4182
4183 (define_expand "mov<mode>"
4184 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4185 (match_operand:VSTRUCT 1 "general_operand" ""))]
4186 "TARGET_SIMD"
4187 {
4188 if (can_create_pseudo_p ())
4189 {
4190 if (GET_CODE (operands[0]) != REG)
4191 operands[1] = force_reg (<MODE>mode, operands[1]);
4192 }
4193 })
4194
4195 (define_insn "*aarch64_mov<mode>"
4196 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4197 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4198 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4199 && (register_operand (operands[0], <MODE>mode)
4200 || register_operand (operands[1], <MODE>mode))"
4201 "@
4202 #
4203 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4204 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4205 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4206 neon_load<nregs>_<nregs>reg_q")
4207 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4208 )
4209
4210 (define_insn "aarch64_be_ld1<mode>"
4211 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4212 (unspec:VALLDI [(match_operand:VALLDI 1 "aarch64_simd_struct_operand" "Utv")]
4213 UNSPEC_LD1))]
4214 "TARGET_SIMD"
4215 "ld1\\t{%0<Vmtype>}, %1"
4216 [(set_attr "type" "neon_load1_1reg<q>")]
4217 )
4218
4219 (define_insn "aarch64_be_st1<mode>"
4220 [(set (match_operand:VALLDI 0 "aarch64_simd_struct_operand" "=Utv")
4221 (unspec:VALLDI [(match_operand:VALLDI 1 "register_operand" "w")]
4222 UNSPEC_ST1))]
4223 "TARGET_SIMD"
4224 "st1\\t{%1<Vmtype>}, %0"
4225 [(set_attr "type" "neon_store1_1reg<q>")]
4226 )
4227
4228 (define_insn "*aarch64_be_movoi"
4229 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4230 (match_operand:OI 1 "general_operand" " w,w,m"))]
4231 "TARGET_SIMD && BYTES_BIG_ENDIAN
4232 && (register_operand (operands[0], OImode)
4233 || register_operand (operands[1], OImode))"
4234 "@
4235 #
4236 stp\\t%q1, %R1, %0
4237 ldp\\t%q0, %R0, %1"
4238 [(set_attr "type" "multiple,neon_store2_2reg_q,neon_load2_2reg_q")
4239 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4240 )
4241
4242 (define_insn "*aarch64_be_movci"
4243 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4244 (match_operand:CI 1 "general_operand" " w,w,o"))]
4245 "TARGET_SIMD && BYTES_BIG_ENDIAN
4246 && (register_operand (operands[0], CImode)
4247 || register_operand (operands[1], CImode))"
4248 "#"
4249 [(set_attr "type" "multiple")
4250 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4251 )
4252
4253 (define_insn "*aarch64_be_movxi"
4254 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4255 (match_operand:XI 1 "general_operand" " w,w,o"))]
4256 "TARGET_SIMD && BYTES_BIG_ENDIAN
4257 && (register_operand (operands[0], XImode)
4258 || register_operand (operands[1], XImode))"
4259 "#"
4260 [(set_attr "type" "multiple")
4261 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4262 )
4263
4264 (define_split
4265 [(set (match_operand:OI 0 "register_operand")
4266 (match_operand:OI 1 "register_operand"))]
4267 "TARGET_SIMD && reload_completed"
4268 [(const_int 0)]
4269 {
4270 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4271 DONE;
4272 })
4273
4274 (define_split
4275 [(set (match_operand:CI 0 "nonimmediate_operand")
4276 (match_operand:CI 1 "general_operand"))]
4277 "TARGET_SIMD && reload_completed"
4278 [(const_int 0)]
4279 {
4280 if (register_operand (operands[0], CImode)
4281 && register_operand (operands[1], CImode))
4282 {
4283 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4284 DONE;
4285 }
4286 else if (BYTES_BIG_ENDIAN)
4287 {
4288 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4289 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4290 emit_move_insn (gen_lowpart (V16QImode,
4291 simplify_gen_subreg (TImode, operands[0],
4292 CImode, 32)),
4293 gen_lowpart (V16QImode,
4294 simplify_gen_subreg (TImode, operands[1],
4295 CImode, 32)));
4296 DONE;
4297 }
4298 else
4299 FAIL;
4300 })
4301
4302 (define_split
4303 [(set (match_operand:XI 0 "nonimmediate_operand")
4304 (match_operand:XI 1 "general_operand"))]
4305 "TARGET_SIMD && reload_completed"
4306 [(const_int 0)]
4307 {
4308 if (register_operand (operands[0], XImode)
4309 && register_operand (operands[1], XImode))
4310 {
4311 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4312 DONE;
4313 }
4314 else if (BYTES_BIG_ENDIAN)
4315 {
4316 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4317 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4318 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4319 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4320 DONE;
4321 }
4322 else
4323 FAIL;
4324 })
4325
4326 (define_expand "aarch64_ld2r<mode>"
4327 [(match_operand:OI 0 "register_operand" "=w")
4328 (match_operand:DI 1 "register_operand" "w")
4329 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4330 "TARGET_SIMD"
4331 {
4332 machine_mode mode = <V_TWO_ELEM>mode;
4333 rtx mem = gen_rtx_MEM (mode, operands[1]);
4334
4335 emit_insn (gen_aarch64_simd_ld2r<mode> (operands[0], mem));
4336 DONE;
4337 })
4338
4339 (define_expand "aarch64_ld3r<mode>"
4340 [(match_operand:CI 0 "register_operand" "=w")
4341 (match_operand:DI 1 "register_operand" "w")
4342 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4343 "TARGET_SIMD"
4344 {
4345 machine_mode mode = <V_THREE_ELEM>mode;
4346 rtx mem = gen_rtx_MEM (mode, operands[1]);
4347
4348 emit_insn (gen_aarch64_simd_ld3r<mode> (operands[0], mem));
4349 DONE;
4350 })
4351
4352 (define_expand "aarch64_ld4r<mode>"
4353 [(match_operand:XI 0 "register_operand" "=w")
4354 (match_operand:DI 1 "register_operand" "w")
4355 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4356 "TARGET_SIMD"
4357 {
4358 machine_mode mode = <V_FOUR_ELEM>mode;
4359 rtx mem = gen_rtx_MEM (mode, operands[1]);
4360
4361 emit_insn (gen_aarch64_simd_ld4r<mode> (operands[0],mem));
4362 DONE;
4363 })
4364
4365 (define_insn "aarch64_ld2<mode>_dreg"
4366 [(set (match_operand:OI 0 "register_operand" "=w")
4367 (subreg:OI
4368 (vec_concat:<VRL2>
4369 (vec_concat:<VDBL>
4370 (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4371 UNSPEC_LD2)
4372 (vec_duplicate:VD (const_int 0)))
4373 (vec_concat:<VDBL>
4374 (unspec:VD [(match_dup 1)]
4375 UNSPEC_LD2)
4376 (vec_duplicate:VD (const_int 0)))) 0))]
4377 "TARGET_SIMD"
4378 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4379 [(set_attr "type" "neon_load2_2reg<q>")]
4380 )
4381
4382 (define_insn "aarch64_ld2<mode>_dreg"
4383 [(set (match_operand:OI 0 "register_operand" "=w")
4384 (subreg:OI
4385 (vec_concat:<VRL2>
4386 (vec_concat:<VDBL>
4387 (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4388 UNSPEC_LD2)
4389 (const_int 0))
4390 (vec_concat:<VDBL>
4391 (unspec:DX [(match_dup 1)]
4392 UNSPEC_LD2)
4393 (const_int 0))) 0))]
4394 "TARGET_SIMD"
4395 "ld1\\t{%S0.1d - %T0.1d}, %1"
4396 [(set_attr "type" "neon_load1_2reg<q>")]
4397 )
4398
4399 (define_insn "aarch64_ld3<mode>_dreg"
4400 [(set (match_operand:CI 0 "register_operand" "=w")
4401 (subreg:CI
4402 (vec_concat:<VRL3>
4403 (vec_concat:<VRL2>
4404 (vec_concat:<VDBL>
4405 (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4406 UNSPEC_LD3)
4407 (vec_duplicate:VD (const_int 0)))
4408 (vec_concat:<VDBL>
4409 (unspec:VD [(match_dup 1)]
4410 UNSPEC_LD3)
4411 (vec_duplicate:VD (const_int 0))))
4412 (vec_concat:<VDBL>
4413 (unspec:VD [(match_dup 1)]
4414 UNSPEC_LD3)
4415 (vec_duplicate:VD (const_int 0)))) 0))]
4416 "TARGET_SIMD"
4417 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4418 [(set_attr "type" "neon_load3_3reg<q>")]
4419 )
4420
4421 (define_insn "aarch64_ld3<mode>_dreg"
4422 [(set (match_operand:CI 0 "register_operand" "=w")
4423 (subreg:CI
4424 (vec_concat:<VRL3>
4425 (vec_concat:<VRL2>
4426 (vec_concat:<VDBL>
4427 (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4428 UNSPEC_LD3)
4429 (const_int 0))
4430 (vec_concat:<VDBL>
4431 (unspec:DX [(match_dup 1)]
4432 UNSPEC_LD3)
4433 (const_int 0)))
4434 (vec_concat:<VDBL>
4435 (unspec:DX [(match_dup 1)]
4436 UNSPEC_LD3)
4437 (const_int 0))) 0))]
4438 "TARGET_SIMD"
4439 "ld1\\t{%S0.1d - %U0.1d}, %1"
4440 [(set_attr "type" "neon_load1_3reg<q>")]
4441 )
4442
4443 (define_insn "aarch64_ld4<mode>_dreg"
4444 [(set (match_operand:XI 0 "register_operand" "=w")
4445 (subreg:XI
4446 (vec_concat:<VRL4>
4447 (vec_concat:<VRL2>
4448 (vec_concat:<VDBL>
4449 (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4450 UNSPEC_LD4)
4451 (vec_duplicate:VD (const_int 0)))
4452 (vec_concat:<VDBL>
4453 (unspec:VD [(match_dup 1)]
4454 UNSPEC_LD4)
4455 (vec_duplicate:VD (const_int 0))))
4456 (vec_concat:<VRL2>
4457 (vec_concat:<VDBL>
4458 (unspec:VD [(match_dup 1)]
4459 UNSPEC_LD4)
4460 (vec_duplicate:VD (const_int 0)))
4461 (vec_concat:<VDBL>
4462 (unspec:VD [(match_dup 1)]
4463 UNSPEC_LD4)
4464 (vec_duplicate:VD (const_int 0))))) 0))]
4465 "TARGET_SIMD"
4466 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4467 [(set_attr "type" "neon_load4_4reg<q>")]
4468 )
4469
4470 (define_insn "aarch64_ld4<mode>_dreg"
4471 [(set (match_operand:XI 0 "register_operand" "=w")
4472 (subreg:XI
4473 (vec_concat:<VRL4>
4474 (vec_concat:<VRL2>
4475 (vec_concat:<VDBL>
4476 (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4477 UNSPEC_LD4)
4478 (const_int 0))
4479 (vec_concat:<VDBL>
4480 (unspec:DX [(match_dup 1)]
4481 UNSPEC_LD4)
4482 (const_int 0)))
4483 (vec_concat:<VRL2>
4484 (vec_concat:<VDBL>
4485 (unspec:DX [(match_dup 1)]
4486 UNSPEC_LD4)
4487 (const_int 0))
4488 (vec_concat:<VDBL>
4489 (unspec:DX [(match_dup 1)]
4490 UNSPEC_LD4)
4491 (const_int 0)))) 0))]
4492 "TARGET_SIMD"
4493 "ld1\\t{%S0.1d - %V0.1d}, %1"
4494 [(set_attr "type" "neon_load1_4reg<q>")]
4495 )
4496
4497 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4498 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4499 (match_operand:DI 1 "register_operand" "r")
4500 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4501 "TARGET_SIMD"
4502 {
4503 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4504 rtx mem = gen_rtx_MEM (mode, operands[1]);
4505
4506 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4507 DONE;
4508 })
4509
4510 (define_expand "aarch64_ld1<VALL:mode>"
4511 [(match_operand:VALL 0 "register_operand")
4512 (match_operand:DI 1 "register_operand")]
4513 "TARGET_SIMD"
4514 {
4515 machine_mode mode = <VALL:MODE>mode;
4516 rtx mem = gen_rtx_MEM (mode, operands[1]);
4517
4518 if (BYTES_BIG_ENDIAN)
4519 emit_insn (gen_aarch64_be_ld1<VALL:mode> (operands[0], mem));
4520 else
4521 emit_move_insn (operands[0], mem);
4522 DONE;
4523 })
4524
4525 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4526 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4527 (match_operand:DI 1 "register_operand" "r")
4528 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4529 "TARGET_SIMD"
4530 {
4531 machine_mode mode = <VSTRUCT:MODE>mode;
4532 rtx mem = gen_rtx_MEM (mode, operands[1]);
4533
4534 emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
4535 DONE;
4536 })
4537
4538 (define_expand "aarch64_ld2_lane<mode>"
4539 [(match_operand:OI 0 "register_operand" "=w")
4540 (match_operand:DI 1 "register_operand" "w")
4541 (match_operand:OI 2 "register_operand" "0")
4542 (match_operand:SI 3 "immediate_operand" "i")
4543 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4544 "TARGET_SIMD"
4545 {
4546 machine_mode mode = <V_TWO_ELEM>mode;
4547 rtx mem = gen_rtx_MEM (mode, operands[1]);
4548
4549 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4550 NULL);
4551 emit_insn (gen_aarch64_vec_load_lanesoi_lane<mode> (operands[0],
4552 mem,
4553 operands[2],
4554 operands[3]));
4555 DONE;
4556 })
4557
4558 (define_expand "aarch64_ld3_lane<mode>"
4559 [(match_operand:CI 0 "register_operand" "=w")
4560 (match_operand:DI 1 "register_operand" "w")
4561 (match_operand:CI 2 "register_operand" "0")
4562 (match_operand:SI 3 "immediate_operand" "i")
4563 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4564 "TARGET_SIMD"
4565 {
4566 machine_mode mode = <V_THREE_ELEM>mode;
4567 rtx mem = gen_rtx_MEM (mode, operands[1]);
4568
4569 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4570 NULL);
4571 emit_insn (gen_aarch64_vec_load_lanesci_lane<mode> (operands[0],
4572 mem,
4573 operands[2],
4574 operands[3]));
4575 DONE;
4576 })
4577
4578 (define_expand "aarch64_ld4_lane<mode>"
4579 [(match_operand:XI 0 "register_operand" "=w")
4580 (match_operand:DI 1 "register_operand" "w")
4581 (match_operand:XI 2 "register_operand" "0")
4582 (match_operand:SI 3 "immediate_operand" "i")
4583 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4584 "TARGET_SIMD"
4585 {
4586 machine_mode mode = <V_FOUR_ELEM>mode;
4587 rtx mem = gen_rtx_MEM (mode, operands[1]);
4588
4589 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4590 NULL);
4591 emit_insn (gen_aarch64_vec_load_lanesxi_lane<mode> (operands[0],
4592 mem,
4593 operands[2],
4594 operands[3]));
4595 DONE;
4596 })
4597
4598
4599
4600 ;; Expanders for builtins to extract vector registers from large
4601 ;; opaque integer modes.
4602
4603 ;; D-register list.
4604
4605 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4606 [(match_operand:VDC 0 "register_operand" "=w")
4607 (match_operand:VSTRUCT 1 "register_operand" "w")
4608 (match_operand:SI 2 "immediate_operand" "i")]
4609 "TARGET_SIMD"
4610 {
4611 int part = INTVAL (operands[2]);
4612 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4613 int offset = part * 16;
4614
4615 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4616 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4617 DONE;
4618 })
4619
4620 ;; Q-register list.
4621
4622 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4623 [(match_operand:VQ 0 "register_operand" "=w")
4624 (match_operand:VSTRUCT 1 "register_operand" "w")
4625 (match_operand:SI 2 "immediate_operand" "i")]
4626 "TARGET_SIMD"
4627 {
4628 int part = INTVAL (operands[2]);
4629 int offset = part * 16;
4630
4631 emit_move_insn (operands[0],
4632 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4633 DONE;
4634 })
4635
4636 ;; Permuted-store expanders for neon intrinsics.
4637
4638 ;; Permute instructions
4639
4640 ;; vec_perm support
4641
4642 (define_expand "vec_perm_const<mode>"
4643 [(match_operand:VALL 0 "register_operand")
4644 (match_operand:VALL 1 "register_operand")
4645 (match_operand:VALL 2 "register_operand")
4646 (match_operand:<V_cmp_result> 3)]
4647 "TARGET_SIMD"
4648 {
4649 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4650 operands[2], operands[3]))
4651 DONE;
4652 else
4653 FAIL;
4654 })
4655
4656 (define_expand "vec_perm<mode>"
4657 [(match_operand:VB 0 "register_operand")
4658 (match_operand:VB 1 "register_operand")
4659 (match_operand:VB 2 "register_operand")
4660 (match_operand:VB 3 "register_operand")]
4661 "TARGET_SIMD"
4662 {
4663 aarch64_expand_vec_perm (operands[0], operands[1],
4664 operands[2], operands[3]);
4665 DONE;
4666 })
4667
4668 (define_insn "aarch64_tbl1<mode>"
4669 [(set (match_operand:VB 0 "register_operand" "=w")
4670 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4671 (match_operand:VB 2 "register_operand" "w")]
4672 UNSPEC_TBL))]
4673 "TARGET_SIMD"
4674 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
4675 [(set_attr "type" "neon_tbl1<q>")]
4676 )
4677
4678 ;; Two source registers.
4679
4680 (define_insn "aarch64_tbl2v16qi"
4681 [(set (match_operand:V16QI 0 "register_operand" "=w")
4682 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4683 (match_operand:V16QI 2 "register_operand" "w")]
4684 UNSPEC_TBL))]
4685 "TARGET_SIMD"
4686 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
4687 [(set_attr "type" "neon_tbl2_q")]
4688 )
4689
4690 (define_insn_and_split "aarch64_combinev16qi"
4691 [(set (match_operand:OI 0 "register_operand" "=w")
4692 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4693 (match_operand:V16QI 2 "register_operand" "w")]
4694 UNSPEC_CONCAT))]
4695 "TARGET_SIMD"
4696 "#"
4697 "&& reload_completed"
4698 [(const_int 0)]
4699 {
4700 aarch64_split_combinev16qi (operands);
4701 DONE;
4702 }
4703 [(set_attr "type" "multiple")]
4704 )
4705
4706 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4707 [(set (match_operand:VALL 0 "register_operand" "=w")
4708 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4709 (match_operand:VALL 2 "register_operand" "w")]
4710 PERMUTE))]
4711 "TARGET_SIMD"
4712 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4713 [(set_attr "type" "neon_permute<q>")]
4714 )
4715
4716 ;; Note immediate (third) operand is lane index not byte index.
4717 (define_insn "aarch64_ext<mode>"
4718 [(set (match_operand:VALL 0 "register_operand" "=w")
4719 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4720 (match_operand:VALL 2 "register_operand" "w")
4721 (match_operand:SI 3 "immediate_operand" "i")]
4722 UNSPEC_EXT))]
4723 "TARGET_SIMD"
4724 {
4725 operands[3] = GEN_INT (INTVAL (operands[3])
4726 * GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
4727 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
4728 }
4729 [(set_attr "type" "neon_ext<q>")]
4730 )
4731
4732 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
4733 [(set (match_operand:VALL 0 "register_operand" "=w")
4734 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
4735 REVERSE))]
4736 "TARGET_SIMD"
4737 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
4738 [(set_attr "type" "neon_rev<q>")]
4739 )
4740
4741 (define_insn "aarch64_st2<mode>_dreg"
4742 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4743 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4744 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4745 UNSPEC_ST2))]
4746 "TARGET_SIMD"
4747 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4748 [(set_attr "type" "neon_store2_2reg")]
4749 )
4750
4751 (define_insn "aarch64_st2<mode>_dreg"
4752 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4753 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4754 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4755 UNSPEC_ST2))]
4756 "TARGET_SIMD"
4757 "st1\\t{%S1.1d - %T1.1d}, %0"
4758 [(set_attr "type" "neon_store1_2reg")]
4759 )
4760
4761 (define_insn "aarch64_st3<mode>_dreg"
4762 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4763 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4764 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4765 UNSPEC_ST3))]
4766 "TARGET_SIMD"
4767 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4768 [(set_attr "type" "neon_store3_3reg")]
4769 )
4770
4771 (define_insn "aarch64_st3<mode>_dreg"
4772 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4773 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4774 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4775 UNSPEC_ST3))]
4776 "TARGET_SIMD"
4777 "st1\\t{%S1.1d - %U1.1d}, %0"
4778 [(set_attr "type" "neon_store1_3reg")]
4779 )
4780
4781 (define_insn "aarch64_st4<mode>_dreg"
4782 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4783 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4784 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4785 UNSPEC_ST4))]
4786 "TARGET_SIMD"
4787 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4788 [(set_attr "type" "neon_store4_4reg")]
4789 )
4790
4791 (define_insn "aarch64_st4<mode>_dreg"
4792 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4793 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4794 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795 UNSPEC_ST4))]
4796 "TARGET_SIMD"
4797 "st1\\t{%S1.1d - %V1.1d}, %0"
4798 [(set_attr "type" "neon_store1_4reg")]
4799 )
4800
4801 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4802 [(match_operand:DI 0 "register_operand" "r")
4803 (match_operand:VSTRUCT 1 "register_operand" "w")
4804 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4805 "TARGET_SIMD"
4806 {
4807 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4808 rtx mem = gen_rtx_MEM (mode, operands[0]);
4809
4810 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4811 DONE;
4812 })
4813
4814 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4815 [(match_operand:DI 0 "register_operand" "r")
4816 (match_operand:VSTRUCT 1 "register_operand" "w")
4817 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4818 "TARGET_SIMD"
4819 {
4820 machine_mode mode = <VSTRUCT:MODE>mode;
4821 rtx mem = gen_rtx_MEM (mode, operands[0]);
4822
4823 emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4824 DONE;
4825 })
4826
4827 (define_expand "aarch64_st2_lane<VQ:mode>"
4828 [(match_operand:DI 0 "register_operand" "r")
4829 (match_operand:OI 1 "register_operand" "w")
4830 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4831 (match_operand:SI 2 "immediate_operand")]
4832 "TARGET_SIMD"
4833 {
4834 machine_mode mode = <V_TWO_ELEM>mode;
4835 rtx mem = gen_rtx_MEM (mode, operands[0]);
4836 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4837
4838 emit_insn (gen_vec_store_lanesoi_lane<VQ:mode> (mem,
4839 operands[1],
4840 operands[2]));
4841 DONE;
4842 })
4843
4844 (define_expand "aarch64_st3_lane<VQ:mode>"
4845 [(match_operand:DI 0 "register_operand" "r")
4846 (match_operand:CI 1 "register_operand" "w")
4847 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4848 (match_operand:SI 2 "immediate_operand")]
4849 "TARGET_SIMD"
4850 {
4851 machine_mode mode = <V_THREE_ELEM>mode;
4852 rtx mem = gen_rtx_MEM (mode, operands[0]);
4853 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4854
4855 emit_insn (gen_vec_store_lanesci_lane<VQ:mode> (mem,
4856 operands[1],
4857 operands[2]));
4858 DONE;
4859 })
4860
4861 (define_expand "aarch64_st4_lane<VQ:mode>"
4862 [(match_operand:DI 0 "register_operand" "r")
4863 (match_operand:XI 1 "register_operand" "w")
4864 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4865 (match_operand:SI 2 "immediate_operand")]
4866 "TARGET_SIMD"
4867 {
4868 machine_mode mode = <V_FOUR_ELEM>mode;
4869 rtx mem = gen_rtx_MEM (mode, operands[0]);
4870 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4871
4872 emit_insn (gen_vec_store_lanesxi_lane<VQ:mode> (mem,
4873 operands[1],
4874 operands[2]));
4875 DONE;
4876 })
4877
4878 (define_expand "aarch64_st1<VALL:mode>"
4879 [(match_operand:DI 0 "register_operand")
4880 (match_operand:VALL 1 "register_operand")]
4881 "TARGET_SIMD"
4882 {
4883 machine_mode mode = <VALL:MODE>mode;
4884 rtx mem = gen_rtx_MEM (mode, operands[0]);
4885
4886 if (BYTES_BIG_ENDIAN)
4887 emit_insn (gen_aarch64_be_st1<VALL:mode> (mem, operands[1]));
4888 else
4889 emit_move_insn (mem, operands[1]);
4890 DONE;
4891 })
4892
4893 ;; Expander for builtins to insert vector registers into large
4894 ;; opaque integer modes.
4895
4896 ;; Q-register list. We don't need a D-reg inserter as we zero
4897 ;; extend them in arm_neon.h and insert the resulting Q-regs.
4898
4899 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4900 [(match_operand:VSTRUCT 0 "register_operand" "+w")
4901 (match_operand:VSTRUCT 1 "register_operand" "0")
4902 (match_operand:VQ 2 "register_operand" "w")
4903 (match_operand:SI 3 "immediate_operand" "i")]
4904 "TARGET_SIMD"
4905 {
4906 int part = INTVAL (operands[3]);
4907 int offset = part * 16;
4908
4909 emit_move_insn (operands[0], operands[1]);
4910 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4911 operands[2]);
4912 DONE;
4913 })
4914
4915 ;; Standard pattern name vec_init<mode>.
4916
4917 (define_expand "vec_init<mode>"
4918 [(match_operand:VALL 0 "register_operand" "")
4919 (match_operand 1 "" "")]
4920 "TARGET_SIMD"
4921 {
4922 aarch64_expand_vector_init (operands[0], operands[1]);
4923 DONE;
4924 })
4925
4926 (define_insn "*aarch64_simd_ld1r<mode>"
4927 [(set (match_operand:VALL 0 "register_operand" "=w")
4928 (vec_duplicate:VALL
4929 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4930 "TARGET_SIMD"
4931 "ld1r\\t{%0.<Vtype>}, %1"
4932 [(set_attr "type" "neon_load1_all_lanes")]
4933 )
4934
4935 (define_insn "aarch64_frecpe<mode>"
4936 [(set (match_operand:VDQF 0 "register_operand" "=w")
4937 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4938 UNSPEC_FRECPE))]
4939 "TARGET_SIMD"
4940 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4941 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
4942 )
4943
4944 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4945 [(set (match_operand:GPF 0 "register_operand" "=w")
4946 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4947 FRECP))]
4948 "TARGET_SIMD"
4949 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
4950 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
4951 )
4952
4953 (define_insn "aarch64_frecps<mode>"
4954 [(set (match_operand:VALLF 0 "register_operand" "=w")
4955 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4956 (match_operand:VALLF 2 "register_operand" "w")]
4957 UNSPEC_FRECPS))]
4958 "TARGET_SIMD"
4959 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4960 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
4961 )
4962
4963 (define_insn "aarch64_urecpe<mode>"
4964 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
4965 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
4966 UNSPEC_URECPE))]
4967 "TARGET_SIMD"
4968 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
4969 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
4970
4971 ;; Standard pattern name vec_extract<mode>.
4972
4973 (define_expand "vec_extract<mode>"
4974 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
4975 (match_operand:VALL 1 "register_operand" "")
4976 (match_operand:SI 2 "immediate_operand" "")]
4977 "TARGET_SIMD"
4978 {
4979 emit_insn
4980 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
4981 DONE;
4982 })
4983
4984 ;; aes
4985
4986 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
4987 [(set (match_operand:V16QI 0 "register_operand" "=w")
4988 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
4989 (match_operand:V16QI 2 "register_operand" "w")]
4990 CRYPTO_AES))]
4991 "TARGET_SIMD && TARGET_CRYPTO"
4992 "aes<aes_op>\\t%0.16b, %2.16b"
4993 [(set_attr "type" "crypto_aese")]
4994 )
4995
4996 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
4997 [(set (match_operand:V16QI 0 "register_operand" "=w")
4998 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
4999 CRYPTO_AESMC))]
5000 "TARGET_SIMD && TARGET_CRYPTO"
5001 "aes<aesmc_op>\\t%0.16b, %1.16b"
5002 [(set_attr "type" "crypto_aesmc")]
5003 )
5004
5005 ;; sha1
5006
5007 (define_insn "aarch64_crypto_sha1hsi"
5008 [(set (match_operand:SI 0 "register_operand" "=w")
5009 (unspec:SI [(match_operand:SI 1
5010 "register_operand" "w")]
5011 UNSPEC_SHA1H))]
5012 "TARGET_SIMD && TARGET_CRYPTO"
5013 "sha1h\\t%s0, %s1"
5014 [(set_attr "type" "crypto_sha1_fast")]
5015 )
5016
5017 (define_insn "aarch64_crypto_sha1su1v4si"
5018 [(set (match_operand:V4SI 0 "register_operand" "=w")
5019 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5020 (match_operand:V4SI 2 "register_operand" "w")]
5021 UNSPEC_SHA1SU1))]
5022 "TARGET_SIMD && TARGET_CRYPTO"
5023 "sha1su1\\t%0.4s, %2.4s"
5024 [(set_attr "type" "crypto_sha1_fast")]
5025 )
5026
5027 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5028 [(set (match_operand:V4SI 0 "register_operand" "=w")
5029 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5030 (match_operand:SI 2 "register_operand" "w")
5031 (match_operand:V4SI 3 "register_operand" "w")]
5032 CRYPTO_SHA1))]
5033 "TARGET_SIMD && TARGET_CRYPTO"
5034 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5035 [(set_attr "type" "crypto_sha1_slow")]
5036 )
5037
5038 (define_insn "aarch64_crypto_sha1su0v4si"
5039 [(set (match_operand:V4SI 0 "register_operand" "=w")
5040 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5041 (match_operand:V4SI 2 "register_operand" "w")
5042 (match_operand:V4SI 3 "register_operand" "w")]
5043 UNSPEC_SHA1SU0))]
5044 "TARGET_SIMD && TARGET_CRYPTO"
5045 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5046 [(set_attr "type" "crypto_sha1_xor")]
5047 )
5048
5049 ;; sha256
5050
5051 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5052 [(set (match_operand:V4SI 0 "register_operand" "=w")
5053 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5054 (match_operand:V4SI 2 "register_operand" "w")
5055 (match_operand:V4SI 3 "register_operand" "w")]
5056 CRYPTO_SHA256))]
5057 "TARGET_SIMD && TARGET_CRYPTO"
5058 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5059 [(set_attr "type" "crypto_sha256_slow")]
5060 )
5061
5062 (define_insn "aarch64_crypto_sha256su0v4si"
5063 [(set (match_operand:V4SI 0 "register_operand" "=w")
5064 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5065 (match_operand:V4SI 2 "register_operand" "w")]
5066 UNSPEC_SHA256SU0))]
5067 "TARGET_SIMD &&TARGET_CRYPTO"
5068 "sha256su0\\t%0.4s, %2.4s"
5069 [(set_attr "type" "crypto_sha256_fast")]
5070 )
5071
5072 (define_insn "aarch64_crypto_sha256su1v4si"
5073 [(set (match_operand:V4SI 0 "register_operand" "=w")
5074 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5075 (match_operand:V4SI 2 "register_operand" "w")
5076 (match_operand:V4SI 3 "register_operand" "w")]
5077 UNSPEC_SHA256SU1))]
5078 "TARGET_SIMD &&TARGET_CRYPTO"
5079 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5080 [(set_attr "type" "crypto_sha256_slow")]
5081 )
5082
5083 ;; pmull
5084
5085 (define_insn "aarch64_crypto_pmulldi"
5086 [(set (match_operand:TI 0 "register_operand" "=w")
5087 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5088 (match_operand:DI 2 "register_operand" "w")]
5089 UNSPEC_PMULL))]
5090 "TARGET_SIMD && TARGET_CRYPTO"
5091 "pmull\\t%0.1q, %1.1d, %2.1d"
5092 [(set_attr "type" "neon_mul_d_long")]
5093 )
5094
5095 (define_insn "aarch64_crypto_pmullv2di"
5096 [(set (match_operand:TI 0 "register_operand" "=w")
5097 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5098 (match_operand:V2DI 2 "register_operand" "w")]
5099 UNSPEC_PMULL2))]
5100 "TARGET_SIMD && TARGET_CRYPTO"
5101 "pmull2\\t%0.1q, %1.2d, %2.2d"
5102 [(set_attr "type" "neon_mul_d_long")]
5103 )
Mirror of https://gcc.gnu.org/git/gcc.git