]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/config/aarch64/aarch64-simd.md
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
gcc/
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64-simd.md
CommitLineData
43e9d192 1;; Machine description for AArch64 AdvSIMD architecture.
5624e564 2;; Copyright (C) 2011-2015 Free Software Foundation, Inc.
43e9d192
IB		 3;; Contributed by ARM Ltd.
4;;
5;; This file is part of GCC.
6;;
7;; GCC is free software; you can redistribute it and/or modify it
8;; under the terms of the GNU General Public License as published by
9;; the Free Software Foundation; either version 3, or (at your option)
10;; any later version.
11;;
12;; GCC is distributed in the hope that it will be useful, but
13;; WITHOUT ANY WARRANTY; without even the implied warranty of
14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15;; General Public License for more details.
16;;
17;; You should have received a copy of the GNU General Public License
18;; along with GCC; see the file COPYING3. If not see
19;; <http://www.gnu.org/licenses/>.
20
43e9d192 21(define_expand "mov<mode>"
348d4b0a
BC		 22 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
23 (match_operand:VALL 1 "general_operand" ""))]
43e9d192
IB		 24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29)
30
31(define_expand "movmisalign<mode>"
348d4b0a
BC		 32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
43e9d192
IB		 34 "TARGET_SIMD"
35{
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42})
43
44(define_insn "aarch64_simd_dup<mode>"
a844a695
AL		 45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ_I
47 (match_operand:<VEL> 1 "register_operand" "r, w")))]
43e9d192 48 "TARGET_SIMD"
91bd4114
JG		 49 "@
50 dup\\t%0.<Vtype>, %<vw>1
51 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
78ec3036 52 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
91bd4114
JG		 53)
54
55(define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF 0 "register_operand" "=w")
57 (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
58 "TARGET_SIMD"
59 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
78ec3036 60 [(set_attr "type" "neon_dup<q>")]
43e9d192
IB		 61)
62
63(define_insn "aarch64_dup_lane<mode>"
91bd4114
JG		 64 [(set (match_operand:VALL 0 "register_operand" "=w")
65 (vec_duplicate:VALL
43e9d192 66 (vec_select:<VEL>
91bd4114 67 (match_operand:VALL 1 "register_operand" "w")
43e9d192
IB		 68 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
69 )))]
70 "TARGET_SIMD"
1dd055a2
JG		 71 {
72 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
73 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
74 }
78ec3036 75 [(set_attr "type" "neon_dup<q>")]
43e9d192
IB		 76)
77
91bd4114
JG		 78(define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
79 [(set (match_operand:VALL 0 "register_operand" "=w")
80 (vec_duplicate:VALL
81 (vec_select:<VEL>
82 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
83 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
84 )))]
43e9d192 85 "TARGET_SIMD"
1dd055a2
JG		 86 {
87 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
88 INTVAL (operands[2])));
89 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
90 }
78ec3036 91 [(set_attr "type" "neon_dup<q>")]
43e9d192
IB		 92)
93
94(define_insn "*aarch64_simd_mov<mode>"
348d4b0a 95 [(set (match_operand:VD 0 "nonimmediate_operand"
635ad37e 96 "=w, m, w, ?r, ?w, ?r, w")
348d4b0a 97 (match_operand:VD 1 "general_operand"
635ad37e 98 "m, w, w, w, r, r, Dn"))]
43e9d192
IB		 99 "TARGET_SIMD
100 && (register_operand (operands[0], <MODE>mode)
101 || register_operand (operands[1], <MODE>mode))"
102{
103 switch (which_alternative)
104 {
635ad37e
TB		 105 case 0: return "ldr\\t%d0, %1";
106 case 1: return "str\\t%d1, %0";
43e9d192
IB		 107 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
108 case 3: return "umov\t%0, %1.d[0]";
109 case 4: return "ins\t%0.d[0], %1";
110 case 5: return "mov\t%0, %1";
111 case 6:
81c2dfb9 112 return aarch64_output_simd_mov_immediate (operands[1],
3520f7cc 113 <MODE>mode, 64);
43e9d192
IB		 114 default: gcc_unreachable ();
115 }
116}
78ec3036 117 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
0f686aa9 118 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
78ec3036 119 mov_reg, neon_move<q>")]
43e9d192
IB		 120)
121
122(define_insn "*aarch64_simd_mov<mode>"
348d4b0a 123 [(set (match_operand:VQ 0 "nonimmediate_operand"
635ad37e 124 "=w, m, w, ?r, ?w, ?r, w")
348d4b0a 125 (match_operand:VQ 1 "general_operand"
635ad37e 126 "m, w, w, w, r, r, Dn"))]
43e9d192
IB		 127 "TARGET_SIMD
128 && (register_operand (operands[0], <MODE>mode)
129 || register_operand (operands[1], <MODE>mode))"
130{
3520f7cc
JG		 131 switch (which_alternative)
132 {
c59b7e28 133 case 0:
635ad37e 134 return "ldr\\t%q0, %1";
c59b7e28 135 case 1:
635ad37e 136 return "str\\t%q1, %0";
c59b7e28
SN		 137 case 2:
138 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
139 case 3:
140 case 4:
141 case 5:
142 return "#";
3520f7cc 143 case 6:
81c2dfb9 144 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
c59b7e28
SN		 145 default:
146 gcc_unreachable ();
3520f7cc 147 }
43e9d192 148}
78ec3036 149 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
0f686aa9
JG		 150 neon_logic<q>, multiple, multiple, multiple,\
151 neon_move<q>")
43e9d192
IB		 152 (set_attr "length" "4,4,4,8,8,8,4")]
153)
154
155(define_split
156 [(set (match_operand:VQ 0 "register_operand" "")
157 (match_operand:VQ 1 "register_operand" ""))]
158 "TARGET_SIMD && reload_completed
159 && GP_REGNUM_P (REGNO (operands[0]))
160 && GP_REGNUM_P (REGNO (operands[1]))"
2d8c6dc1 161 [(const_int 0)]
43e9d192 162{
2d8c6dc1
AH		 163 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
164 DONE;
43e9d192
IB		 165})
166
fd4842cd
SN		 167(define_split
168 [(set (match_operand:VQ 0 "register_operand" "")
169 (match_operand:VQ 1 "register_operand" ""))]
170 "TARGET_SIMD && reload_completed
171 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
172 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
173 [(const_int 0)]
174{
175 aarch64_split_simd_move (operands[0], operands[1]);
176 DONE;
177})
178
c59b7e28 179(define_expand "aarch64_split_simd_mov<mode>"
fd4842cd
SN		 180 [(set (match_operand:VQ 0)
181 (match_operand:VQ 1))]
182 "TARGET_SIMD"
183 {
184 rtx dst = operands[0];
185 rtx src = operands[1];
186
187 if (GP_REGNUM_P (REGNO (src)))
188 {
c59b7e28
SN		 189 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
190 rtx src_high_part = gen_highpart (<VHALF>mode, src);
fd4842cd
SN		 191
192 emit_insn
c59b7e28 193 (gen_move_lo_quad_<mode> (dst, src_low_part));
fd4842cd 194 emit_insn
c59b7e28 195 (gen_move_hi_quad_<mode> (dst, src_high_part));
fd4842cd
SN		 196 }
197
198 else
199 {
c59b7e28
SN		 200 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
201 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
202 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
203 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
fd4842cd
SN		 204
205 emit_insn
c59b7e28 206 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
fd4842cd 207 emit_insn
c59b7e28 208 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
fd4842cd
SN		 209 }
210 DONE;
211 }
212)
213
fd4842cd
SN		 214(define_insn "aarch64_simd_mov_from_<mode>low"
215 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
216 (vec_select:<VHALF>
217 (match_operand:VQ 1 "register_operand" "w")
218 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
219 "TARGET_SIMD && reload_completed"
220 "umov\t%0, %1.d[0]"
78ec3036 221 [(set_attr "type" "neon_to_gp<q>")
fd4842cd
SN		 222 (set_attr "length" "4")
223 ])
224
225(define_insn "aarch64_simd_mov_from_<mode>high"
226 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
227 (vec_select:<VHALF>
228 (match_operand:VQ 1 "register_operand" "w")
229 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
230 "TARGET_SIMD && reload_completed"
231 "umov\t%0, %1.d[1]"
78ec3036 232 [(set_attr "type" "neon_to_gp<q>")
fd4842cd
SN		 233 (set_attr "length" "4")
234 ])
235
43e9d192 236(define_insn "orn<mode>3"
a844a695
AL		 237 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
238 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
239 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 240 "TARGET_SIMD"
241 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
78ec3036 242 [(set_attr "type" "neon_logic<q>")]
43e9d192
IB		 243)
244
245(define_insn "bic<mode>3"
a844a695
AL		 246 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
247 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
248 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 249 "TARGET_SIMD"
250 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
78ec3036 251 [(set_attr "type" "neon_logic<q>")]
43e9d192
IB		 252)
253
254(define_insn "add<mode>3"
a844a695
AL		 255 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
256 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
257 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 258 "TARGET_SIMD"
259 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 260 [(set_attr "type" "neon_add<q>")]
43e9d192
IB		 261)
262
263(define_insn "sub<mode>3"
a844a695
AL		 264 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
265 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
266 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 267 "TARGET_SIMD"
268 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 269 [(set_attr "type" "neon_sub<q>")]
43e9d192
IB		 270)
271
272(define_insn "mul<mode>3"
a844a695
AL		 273 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
274 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
275 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
43e9d192
IB		 276 "TARGET_SIMD"
277 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 278 [(set_attr "type" "neon_mul_<Vetype><q>")]
43e9d192
IB		 279)
280
7eb2bd57 281(define_insn "bswap<mode>2"
c7f28cd5
KT		 282 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
283 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
284 "TARGET_SIMD"
285 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
286 [(set_attr "type" "neon_rev<q>")]
287)
288
cf465d71
AL		 289(define_insn "aarch64_rbit<mode>"
290 [(set (match_operand:VB 0 "register_operand" "=w")
291 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
292 UNSPEC_RBIT))]
293 "TARGET_SIMD"
294 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
295 [(set_attr "type" "neon_rbit")]
296)
297
5e32e83b
JW		 298(define_expand "ctz<mode>2"
299 [(set (match_operand:VS 0 "register_operand")
300 (ctz:VS (match_operand:VS 1 "register_operand")))]
301 "TARGET_SIMD"
302 {
7eb2bd57 303 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
5e32e83b
JW		 304 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
305 <MODE>mode, 0);
306 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
307 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
308 DONE;
309 }
310)
311
779aea46
JG		 312(define_insn "*aarch64_mul3_elt<mode>"
313 [(set (match_operand:VMUL 0 "register_operand" "=w")
314 (mult:VMUL
315 (vec_duplicate:VMUL
316 (vec_select:<VEL>
317 (match_operand:VMUL 1 "register_operand" "<h_con>")
318 (parallel [(match_operand:SI 2 "immediate_operand")])))
319 (match_operand:VMUL 3 "register_operand" "w")))]
320 "TARGET_SIMD"
1dd055a2
JG		 321 {
322 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
323 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
324 }
78ec3036 325 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
779aea46
JG		 326)
327
328(define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
329 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
330 (mult:VMUL_CHANGE_NLANES
331 (vec_duplicate:VMUL_CHANGE_NLANES
332 (vec_select:<VEL>
333 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
334 (parallel [(match_operand:SI 2 "immediate_operand")])))
335 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
336 "TARGET_SIMD"
1dd055a2
JG		 337 {
338 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
339 INTVAL (operands[2])));
340 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
341 }
78ec3036 342 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
779aea46
JG		 343)
344
345(define_insn "*aarch64_mul3_elt_to_128df"
346 [(set (match_operand:V2DF 0 "register_operand" "=w")
347 (mult:V2DF
348 (vec_duplicate:V2DF
349 (match_operand:DF 2 "register_operand" "w"))
350 (match_operand:V2DF 1 "register_operand" "w")))]
351 "TARGET_SIMD"
352 "fmul\\t%0.2d, %1.2d, %2.d[0]"
78ec3036 353 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
779aea46
JG		 354)
355
356(define_insn "*aarch64_mul3_elt_to_64v2df"
357 [(set (match_operand:DF 0 "register_operand" "=w")
358 (mult:DF
359 (vec_select:DF
360 (match_operand:V2DF 1 "register_operand" "w")
361 (parallel [(match_operand:SI 2 "immediate_operand")]))
362 (match_operand:DF 3 "register_operand" "w")))]
363 "TARGET_SIMD"
1dd055a2
JG		 364 {
365 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
366 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
367 }
78ec3036 368 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
779aea46
JG		 369)
370
43e9d192 371(define_insn "neg<mode>2"
a844a695
AL		 372 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
373 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
43e9d192
IB		 374 "TARGET_SIMD"
375 "neg\t%0.<Vtype>, %1.<Vtype>"
78ec3036 376 [(set_attr "type" "neon_neg<q>")]
43e9d192
IB		 377)
378
379(define_insn "abs<mode>2"
a844a695
AL		 380 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
381 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
43e9d192
IB		 382 "TARGET_SIMD"
383 "abs\t%0.<Vtype>, %1.<Vtype>"
78ec3036 384 [(set_attr "type" "neon_abs<q>")]
43e9d192
IB		 385)
386
fa2e9a58
N		 387(define_insn "abd<mode>_3"
388 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
389 (abs:VDQ_BHSI (minus:VDQ_BHSI
390 (match_operand:VDQ_BHSI 1 "register_operand" "w")
391 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
392 "TARGET_SIMD"
393 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 394 [(set_attr "type" "neon_abd<q>")]
fa2e9a58
N		 395)
396
397(define_insn "aba<mode>_3"
398 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
399 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
400 (match_operand:VDQ_BHSI 1 "register_operand" "w")
401 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
402 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
403 "TARGET_SIMD"
404 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 405 [(set_attr "type" "neon_arith_acc<q>")]
fa2e9a58
N		 406)
407
408(define_insn "fabd<mode>_3"
409 [(set (match_operand:VDQF 0 "register_operand" "=w")
410 (abs:VDQF (minus:VDQF
411 (match_operand:VDQF 1 "register_operand" "w")
412 (match_operand:VDQF 2 "register_operand" "w"))))]
413 "TARGET_SIMD"
414 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 415 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
fa2e9a58
N		 416)
417
1d0c8e5c
VP		 418(define_insn "*fabd_scalar<mode>3"
419 [(set (match_operand:GPF 0 "register_operand" "=w")
420 (abs:GPF (minus:GPF
421 (match_operand:GPF 1 "register_operand" "w")
422 (match_operand:GPF 2 "register_operand" "w"))))]
423 "TARGET_SIMD"
424 "fabd\t%<s>0, %<s>1, %<s>2"
78ec3036 425 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
1d0c8e5c
VP		 426)
427
43e9d192 428(define_insn "and<mode>3"
a844a695
AL		 429 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
430 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
431 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 432 "TARGET_SIMD"
433 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
78ec3036 434 [(set_attr "type" "neon_logic<q>")]
43e9d192
IB		 435)
436
437(define_insn "ior<mode>3"
a844a695
AL		 438 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
439 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
440 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 441 "TARGET_SIMD"
442 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
78ec3036 443 [(set_attr "type" "neon_logic<q>")]
43e9d192
IB		 444)
445
446(define_insn "xor<mode>3"
a844a695
AL		 447 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
448 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
449 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 450 "TARGET_SIMD"
451 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
78ec3036 452 [(set_attr "type" "neon_logic<q>")]
43e9d192
IB		 453)
454
455(define_insn "one_cmpl<mode>2"
a844a695
AL		 456 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
457 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
43e9d192
IB		 458 "TARGET_SIMD"
459 "not\t%0.<Vbtype>, %1.<Vbtype>"
78ec3036 460 [(set_attr "type" "neon_logic<q>")]
43e9d192
IB		 461)
462
463(define_insn "aarch64_simd_vec_set<mode>"
a844a695
AL		 464 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
465 (vec_merge:VDQ_BHSI
466 (vec_duplicate:VDQ_BHSI
960ceebc 467 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
a844a695 468 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
960ceebc 469 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
43e9d192 470 "TARGET_SIMD"
e58bf20a
TB		 471 {
472 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
473 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
474 switch (which_alternative)
475 {
476 case 0:
477 return "ins\\t%0.<Vetype>[%p2], %w1";
478 case 1:
479 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
960ceebc
AL		 480 case 2:
481 return "ld1\\t{%0.<Vetype>}[%p2], %1";
e58bf20a
TB		 482 default:
483 gcc_unreachable ();
484 }
485 }
960ceebc 486 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_1reg<q>")]
43e9d192
IB		 487)
488
489(define_insn "aarch64_simd_lshr<mode>"
a844a695
AL		 490 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
491 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
492 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
43e9d192
IB		 493 "TARGET_SIMD"
494 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
78ec3036 495 [(set_attr "type" "neon_shift_imm<q>")]
43e9d192
IB		 496)
497
498(define_insn "aarch64_simd_ashr<mode>"
a844a695
AL		 499 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
500 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
501 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
43e9d192
IB		 502 "TARGET_SIMD"
503 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
78ec3036 504 [(set_attr "type" "neon_shift_imm<q>")]
43e9d192
IB		 505)
506
507(define_insn "aarch64_simd_imm_shl<mode>"
a844a695
AL		 508 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
509 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
510 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
43e9d192
IB		 511 "TARGET_SIMD"
512 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
78ec3036 513 [(set_attr "type" "neon_shift_imm<q>")]
43e9d192
IB		 514)
515
516(define_insn "aarch64_simd_reg_sshl<mode>"
a844a695
AL		 517 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
518 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
519 (match_operand:VDQ_I 2 "register_operand" "w")))]
43e9d192
IB		 520 "TARGET_SIMD"
521 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 522 [(set_attr "type" "neon_shift_reg<q>")]
43e9d192
IB		 523)
524
525(define_insn "aarch64_simd_reg_shl<mode>_unsigned"
a844a695
AL		 526 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
527 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
528 (match_operand:VDQ_I 2 "register_operand" "w")]
43e9d192
IB		 529 UNSPEC_ASHIFT_UNSIGNED))]
530 "TARGET_SIMD"
531 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 532 [(set_attr "type" "neon_shift_reg<q>")]
43e9d192
IB		 533)
534
535(define_insn "aarch64_simd_reg_shl<mode>_signed"
a844a695
AL		 536 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
537 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
538 (match_operand:VDQ_I 2 "register_operand" "w")]
43e9d192
IB		 539 UNSPEC_ASHIFT_SIGNED))]
540 "TARGET_SIMD"
541 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 542 [(set_attr "type" "neon_shift_reg<q>")]
43e9d192
IB		 543)
544
545(define_expand "ashl<mode>3"
a844a695
AL		 546 [(match_operand:VDQ_I 0 "register_operand" "")
547 (match_operand:VDQ_I 1 "register_operand" "")
43e9d192
IB		 548 (match_operand:SI 2 "general_operand" "")]
549 "TARGET_SIMD"
550{
551 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
552 int shift_amount;
553
554 if (CONST_INT_P (operands[2]))
555 {
556 shift_amount = INTVAL (operands[2]);
557 if (shift_amount >= 0 && shift_amount < bit_width)
558 {
559 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
560 shift_amount);
561 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
562 operands[1],
563 tmp));
564 DONE;
565 }
566 else
567 {
568 operands[2] = force_reg (SImode, operands[2]);
569 }
570 }
571 else if (MEM_P (operands[2]))
572 {
573 operands[2] = force_reg (SImode, operands[2]);
574 }
575
576 if (REG_P (operands[2]))
577 {
578 rtx tmp = gen_reg_rtx (<MODE>mode);
579 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
580 convert_to_mode (<VEL>mode,
581 operands[2],
582 0)));
583 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
584 tmp));
585 DONE;
586 }
587 else
588 FAIL;
589}
590)
591
592(define_expand "lshr<mode>3"
a844a695
AL		 593 [(match_operand:VDQ_I 0 "register_operand" "")
594 (match_operand:VDQ_I 1 "register_operand" "")
43e9d192
IB		 595 (match_operand:SI 2 "general_operand" "")]
596 "TARGET_SIMD"
597{
598 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
599 int shift_amount;
600
601 if (CONST_INT_P (operands[2]))
602 {
603 shift_amount = INTVAL (operands[2]);
604 if (shift_amount > 0 && shift_amount <= bit_width)
605 {
606 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
607 shift_amount);
608 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
609 operands[1],
610 tmp));
611 DONE;
612 }
613 else
614 operands[2] = force_reg (SImode, operands[2]);
615 }
616 else if (MEM_P (operands[2]))
617 {
618 operands[2] = force_reg (SImode, operands[2]);
619 }
620
621 if (REG_P (operands[2]))
622 {
623 rtx tmp = gen_reg_rtx (SImode);
624 rtx tmp1 = gen_reg_rtx (<MODE>mode);
625 emit_insn (gen_negsi2 (tmp, operands[2]));
626 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
627 convert_to_mode (<VEL>mode,
628 tmp, 0)));
629 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
630 operands[1],
631 tmp1));
632 DONE;
633 }
634 else
635 FAIL;
636}
637)
638
639(define_expand "ashr<mode>3"
a844a695
AL		 640 [(match_operand:VDQ_I 0 "register_operand" "")
641 (match_operand:VDQ_I 1 "register_operand" "")
43e9d192
IB		 642 (match_operand:SI 2 "general_operand" "")]
643 "TARGET_SIMD"
644{
645 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
646 int shift_amount;
647
648 if (CONST_INT_P (operands[2]))
649 {
650 shift_amount = INTVAL (operands[2]);
651 if (shift_amount > 0 && shift_amount <= bit_width)
652 {
653 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
654 shift_amount);
655 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
656 operands[1],
657 tmp));
658 DONE;
659 }
660 else
661 operands[2] = force_reg (SImode, operands[2]);
662 }
663 else if (MEM_P (operands[2]))
664 {
665 operands[2] = force_reg (SImode, operands[2]);
666 }
667
668 if (REG_P (operands[2]))
669 {
670 rtx tmp = gen_reg_rtx (SImode);
671 rtx tmp1 = gen_reg_rtx (<MODE>mode);
672 emit_insn (gen_negsi2 (tmp, operands[2]));
673 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
674 convert_to_mode (<VEL>mode,
675 tmp, 0)));
676 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
677 operands[1],
678 tmp1));
679 DONE;
680 }
681 else
682 FAIL;
683}
684)
685
686(define_expand "vashl<mode>3"
a844a695
AL		 687 [(match_operand:VDQ_I 0 "register_operand" "")
688 (match_operand:VDQ_I 1 "register_operand" "")
689 (match_operand:VDQ_I 2 "register_operand" "")]
43e9d192
IB		 690 "TARGET_SIMD"
691{
692 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
693 operands[2]));
694 DONE;
695})
696
a844a695 697;; Using mode VDQ_BHSI as there is no V2DImode neg!
43e9d192
IB		 698;; Negating individual lanes most certainly offsets the
699;; gain from vectorization.
700(define_expand "vashr<mode>3"
a844a695
AL		 701 [(match_operand:VDQ_BHSI 0 "register_operand" "")
702 (match_operand:VDQ_BHSI 1 "register_operand" "")
703 (match_operand:VDQ_BHSI 2 "register_operand" "")]
43e9d192
IB		 704 "TARGET_SIMD"
705{
706 rtx neg = gen_reg_rtx (<MODE>mode);
707 emit (gen_neg<mode>2 (neg, operands[2]));
708 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
709 neg));
710 DONE;
711})
712
f9a4c9a6
AV		 713;; DI vector shift
714(define_expand "aarch64_ashr_simddi"
715 [(match_operand:DI 0 "register_operand" "=w")
716 (match_operand:DI 1 "register_operand" "w")
179d2941 717 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
f9a4c9a6
AV		 718 "TARGET_SIMD"
719 {
b5b34d37
AL		 720 /* An arithmetic shift right by 64 fills the result with copies of the sign
721 bit, just like asr by 63 - however the standard pattern does not handle
722 a shift by 64. */
f9a4c9a6 723 if (INTVAL (operands[2]) == 64)
b5b34d37
AL		 724 operands[2] = GEN_INT (63);
725 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
f9a4c9a6
AV		 726 DONE;
727 }
728)
729
43e9d192 730(define_expand "vlshr<mode>3"
a844a695
AL		 731 [(match_operand:VDQ_BHSI 0 "register_operand" "")
732 (match_operand:VDQ_BHSI 1 "register_operand" "")
733 (match_operand:VDQ_BHSI 2 "register_operand" "")]
43e9d192
IB		 734 "TARGET_SIMD"
735{
736 rtx neg = gen_reg_rtx (<MODE>mode);
737 emit (gen_neg<mode>2 (neg, operands[2]));
738 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
739 neg));
740 DONE;
741})
742
252c7556
AV		 743(define_expand "aarch64_lshr_simddi"
744 [(match_operand:DI 0 "register_operand" "=w")
745 (match_operand:DI 1 "register_operand" "w")
746 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
747 "TARGET_SIMD"
748 {
749 if (INTVAL (operands[2]) == 64)
84488801 750 emit_move_insn (operands[0], const0_rtx);
252c7556
AV		 751 else
752 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
753 DONE;
754 }
755)
756
43e9d192 757(define_expand "vec_set<mode>"
a844a695 758 [(match_operand:VDQ_BHSI 0 "register_operand")
d22253e5
TB		 759 (match_operand:<VEL> 1 "register_operand")
760 (match_operand:SI 2 "immediate_operand")]
43e9d192
IB		 761 "TARGET_SIMD"
762 {
763 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
764 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
765 GEN_INT (elem), operands[0]));
766 DONE;
767 }
768)
769
0b4eefd5
AL		 770;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
771(define_insn "vec_shr_<mode>"
772 [(set (match_operand:VD 0 "register_operand" "=w")
773 (lshiftrt:VD (match_operand:VD 1 "register_operand" "w")
774 (match_operand:SI 2 "immediate_operand" "i")))]
775 "TARGET_SIMD"
776 {
777 if (BYTES_BIG_ENDIAN)
778 return "ushl %d0, %d1, %2";
779 else
780 return "ushr %d0, %d1, %2";
781 }
782 [(set_attr "type" "neon_shift_imm")]
783)
784
43e9d192 785(define_insn "aarch64_simd_vec_setv2di"
d22253e5 786 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
43e9d192
IB		 787 (vec_merge:V2DI
788 (vec_duplicate:V2DI
d22253e5
TB		 789 (match_operand:DI 1 "register_operand" "r,w"))
790 (match_operand:V2DI 3 "register_operand" "0,0")
791 (match_operand:SI 2 "immediate_operand" "i,i")))]
43e9d192 792 "TARGET_SIMD"
e58bf20a
TB		 793 {
794 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
795 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
796 switch (which_alternative)
797 {
798 case 0:
799 return "ins\\t%0.d[%p2], %1";
800 case 1:
801 return "ins\\t%0.d[%p2], %1.d[0]";
802 default:
803 gcc_unreachable ();
804 }
805 }
78ec3036 806 [(set_attr "type" "neon_from_gp, neon_ins_q")]
43e9d192
IB		 807)
808
809(define_expand "vec_setv2di"
d22253e5
TB		 810 [(match_operand:V2DI 0 "register_operand")
811 (match_operand:DI 1 "register_operand")
812 (match_operand:SI 2 "immediate_operand")]
43e9d192
IB		 813 "TARGET_SIMD"
814 {
815 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
816 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
817 GEN_INT (elem), operands[0]));
818 DONE;
819 }
820)
821
822(define_insn "aarch64_simd_vec_set<mode>"
823 [(set (match_operand:VDQF 0 "register_operand" "=w")
824 (vec_merge:VDQF
825 (vec_duplicate:VDQF
826 (match_operand:<VEL> 1 "register_operand" "w"))
827 (match_operand:VDQF 3 "register_operand" "0")
828 (match_operand:SI 2 "immediate_operand" "i")))]
829 "TARGET_SIMD"
e58bf20a
TB		 830 {
831 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
832
833 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
834 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
835 }
78ec3036 836 [(set_attr "type" "neon_ins<q>")]
43e9d192
IB		 837)
838
839(define_expand "vec_set<mode>"
840 [(match_operand:VDQF 0 "register_operand" "+w")
841 (match_operand:<VEL> 1 "register_operand" "w")
842 (match_operand:SI 2 "immediate_operand" "")]
843 "TARGET_SIMD"
844 {
845 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
846 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
847 GEN_INT (elem), operands[0]));
848 DONE;
849 }
850)
851
852
853(define_insn "aarch64_mla<mode>"
a844a695
AL		 854 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
855 (plus:VDQ_BHSI (mult:VDQ_BHSI
856 (match_operand:VDQ_BHSI 2 "register_operand" "w")
857 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
858 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
43e9d192
IB		 859 "TARGET_SIMD"
860 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
78ec3036 861 [(set_attr "type" "neon_mla_<Vetype><q>")]
43e9d192
IB		 862)
863
828e70c1
JG		 864(define_insn "*aarch64_mla_elt<mode>"
865 [(set (match_operand:VDQHS 0 "register_operand" "=w")
866 (plus:VDQHS
867 (mult:VDQHS
868 (vec_duplicate:VDQHS
869 (vec_select:<VEL>
870 (match_operand:VDQHS 1 "register_operand" "<h_con>")
871 (parallel [(match_operand:SI 2 "immediate_operand")])))
872 (match_operand:VDQHS 3 "register_operand" "w"))
873 (match_operand:VDQHS 4 "register_operand" "0")))]
874 "TARGET_SIMD"
1dd055a2
JG		 875 {
876 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
877 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
878 }
78ec3036 879 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 880)
881
882(define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
883 [(set (match_operand:VDQHS 0 "register_operand" "=w")
884 (plus:VDQHS
885 (mult:VDQHS
886 (vec_duplicate:VDQHS
887 (vec_select:<VEL>
888 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
889 (parallel [(match_operand:SI 2 "immediate_operand")])))
890 (match_operand:VDQHS 3 "register_operand" "w"))
891 (match_operand:VDQHS 4 "register_operand" "0")))]
892 "TARGET_SIMD"
1dd055a2
JG		 893 {
894 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
895 INTVAL (operands[2])));
896 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
897 }
78ec3036 898 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 899)
900
43e9d192 901(define_insn "aarch64_mls<mode>"
a844a695
AL		 902 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
903 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
904 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
905 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
43e9d192
IB		 906 "TARGET_SIMD"
907 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
78ec3036 908 [(set_attr "type" "neon_mla_<Vetype><q>")]
43e9d192
IB		 909)
910
828e70c1
JG		 911(define_insn "*aarch64_mls_elt<mode>"
912 [(set (match_operand:VDQHS 0 "register_operand" "=w")
913 (minus:VDQHS
914 (match_operand:VDQHS 4 "register_operand" "0")
915 (mult:VDQHS
916 (vec_duplicate:VDQHS
917 (vec_select:<VEL>
918 (match_operand:VDQHS 1 "register_operand" "<h_con>")
919 (parallel [(match_operand:SI 2 "immediate_operand")])))
920 (match_operand:VDQHS 3 "register_operand" "w"))))]
921 "TARGET_SIMD"
1dd055a2
JG		 922 {
923 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
924 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
925 }
78ec3036 926 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 927)
928
929(define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
930 [(set (match_operand:VDQHS 0 "register_operand" "=w")
931 (minus:VDQHS
932 (match_operand:VDQHS 4 "register_operand" "0")
933 (mult:VDQHS
934 (vec_duplicate:VDQHS
935 (vec_select:<VEL>
936 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
937 (parallel [(match_operand:SI 2 "immediate_operand")])))
938 (match_operand:VDQHS 3 "register_operand" "w"))))]
939 "TARGET_SIMD"
1dd055a2
JG		 940 {
941 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
942 INTVAL (operands[2])));
943 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
944 }
78ec3036 945 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 946)
947
43e9d192 948;; Max/Min operations.
998eaf97 949(define_insn "<su><maxmin><mode>3"
a844a695
AL		 950 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
951 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
952 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
43e9d192 953 "TARGET_SIMD"
998eaf97 954 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 955 [(set_attr "type" "neon_minmax<q>")]
43e9d192
IB		 956)
957
da9c88d8
RL		 958(define_expand "<su><maxmin>v2di3"
959 [(set (match_operand:V2DI 0 "register_operand" "")
960 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
961 (match_operand:V2DI 2 "register_operand" "")))]
962 "TARGET_SIMD"
963{
964 enum rtx_code cmp_operator;
965 rtx cmp_fmt;
966
967 switch (<CODE>)
968 {
969 case UMIN:
970 cmp_operator = LTU;
971 break;
972 case SMIN:
973 cmp_operator = LT;
974 break;
975 case UMAX:
976 cmp_operator = GTU;
977 break;
978 case SMAX:
979 cmp_operator = GT;
980 break;
981 default:
982 gcc_unreachable ();
983 }
984
985 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
986 emit_insn (gen_aarch64_vcond_internalv2div2di (operands[0], operands[1],
987 operands[2], cmp_fmt, operands[1], operands[2]));
988 DONE;
989})
990
7abab3d1
FY		 991;; Pairwise Integer Max/Min operations.
992(define_insn "aarch64_<maxmin_uns>p<mode>"
993 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
994 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
995 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
996 MAXMINV))]
997 "TARGET_SIMD"
998 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
999 [(set_attr "type" "neon_minmax<q>")]
1000)
1001
1002;; Pairwise FP Max/Min operations.
1003(define_insn "aarch64_<maxmin_uns>p<mode>"
1004 [(set (match_operand:VDQF 0 "register_operand" "=w")
1005 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1006 (match_operand:VDQF 2 "register_operand" "w")]
1007 FMAXMINV))]
1008 "TARGET_SIMD"
1009 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1010 [(set_attr "type" "neon_minmax<q>")]
1011)
1012
5a908485
JG		 1013;; vec_concat gives a new vector with the low elements from operand 1, and
1014;; the high elements from operand 2. That is to say, given op1 = { a, b }
1015;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1016;; What that means, is that the RTL descriptions of the below patterns
1017;; need to change depending on endianness.
43e9d192 1018
5a908485
JG		 1019;; Move to the low architectural bits of the register.
1020;; On little-endian this is { operand, zeroes }
1021;; On big-endian this is { zeroes, operand }
1022
1023(define_insn "move_lo_quad_internal_<mode>"
51437269
GW		 1024 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1025 (vec_concat:VQ_NO2E
c59b7e28 1026 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
43e9d192 1027 (vec_duplicate:<VHALF> (const_int 0))))]
5a908485
JG		 1028 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1029 "@
1030 dup\\t%d0, %1.d[0]
1031 fmov\\t%d0, %1
1032 dup\\t%d0, %1"
1033 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1034 (set_attr "simd" "yes,*,yes")
1035 (set_attr "fp" "*,yes,*")
1036 (set_attr "length" "4")]
1037)
1038
51437269
GW		 1039(define_insn "move_lo_quad_internal_<mode>"
1040 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1041 (vec_concat:VQ_2E
1042 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1043 (const_int 0)))]
1044 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1045 "@
1046 dup\\t%d0, %1.d[0]
1047 fmov\\t%d0, %1
1048 dup\\t%d0, %1"
1049 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1050 (set_attr "simd" "yes,*,yes")
1051 (set_attr "fp" "*,yes,*")
1052 (set_attr "length" "4")]
1053)
1054
5a908485 1055(define_insn "move_lo_quad_internal_be_<mode>"
51437269
GW		 1056 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1057 (vec_concat:VQ_NO2E
5a908485
JG		 1058 (vec_duplicate:<VHALF> (const_int 0))
1059 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1060 "TARGET_SIMD && BYTES_BIG_ENDIAN"
c59b7e28 1061 "@
7af26def
SN		 1062 dup\\t%d0, %1.d[0]
1063 fmov\\t%d0, %1
1064 dup\\t%d0, %1"
d7256bb8 1065 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
c59b7e28
SN		 1066 (set_attr "simd" "yes,*,yes")
1067 (set_attr "fp" "*,yes,*")
1068 (set_attr "length" "4")]
43e9d192
IB		 1069)
1070
51437269
GW		 1071(define_insn "move_lo_quad_internal_be_<mode>"
1072 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1073 (vec_concat:VQ_2E
1074 (const_int 0)
1075 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1076 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1077 "@
1078 dup\\t%d0, %1.d[0]
1079 fmov\\t%d0, %1
1080 dup\\t%d0, %1"
1081 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1082 (set_attr "simd" "yes,*,yes")
1083 (set_attr "fp" "*,yes,*")
1084 (set_attr "length" "4")]
1085)
1086
5a908485
JG		 1087(define_expand "move_lo_quad_<mode>"
1088 [(match_operand:VQ 0 "register_operand")
1089 (match_operand:VQ 1 "register_operand")]
1090 "TARGET_SIMD"
1091{
1092 if (BYTES_BIG_ENDIAN)
1093 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1094 else
1095 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1096 DONE;
1097}
1098)
1099
1100;; Move operand1 to the high architectural bits of the register, keeping
1101;; the low architectural bits of operand2.
1102;; For little-endian this is { operand2, operand1 }
1103;; For big-endian this is { operand1, operand2 }
43e9d192
IB		 1104
1105(define_insn "aarch64_simd_move_hi_quad_<mode>"
c59b7e28 1106 [(set (match_operand:VQ 0 "register_operand" "+w,w")
43e9d192
IB		 1107 (vec_concat:VQ
1108 (vec_select:<VHALF>
1109 (match_dup 0)
1110 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
c59b7e28 1111 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
5a908485 1112 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
c59b7e28
SN		 1113 "@
1114 ins\\t%0.d[1], %1.d[0]
1115 ins\\t%0.d[1], %1"
5a908485
JG		 1116 [(set_attr "type" "neon_ins")]
1117)
1118
1119(define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1120 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1121 (vec_concat:VQ
1122 (match_operand:<VHALF> 1 "register_operand" "w,r")
1123 (vec_select:<VHALF>
1124 (match_dup 0)
988fa693 1125 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
5a908485
JG		 1126 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1127 "@
1128 ins\\t%0.d[1], %1.d[0]
1129 ins\\t%0.d[1], %1"
1130 [(set_attr "type" "neon_ins")]
43e9d192
IB		 1131)
1132
1133(define_expand "move_hi_quad_<mode>"
1134 [(match_operand:VQ 0 "register_operand" "")
1135 (match_operand:<VHALF> 1 "register_operand" "")]
1136 "TARGET_SIMD"
1137{
988fa693 1138 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
5a908485
JG		 1139 if (BYTES_BIG_ENDIAN)
1140 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1141 operands[1], p));
1142 else
1143 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1144 operands[1], p));
43e9d192
IB		 1145 DONE;
1146})
1147
1148;; Narrowing operations.
1149
1150;; For doubles.
1151(define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1152 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1153 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1154 "TARGET_SIMD"
1155 "xtn\\t%0.<Vntype>, %1.<Vtype>"
78ec3036 1156 [(set_attr "type" "neon_shift_imm_narrow_q")]
43e9d192
IB		 1157)
1158
1159(define_expand "vec_pack_trunc_<mode>"
1160 [(match_operand:<VNARROWD> 0 "register_operand" "")
1161 (match_operand:VDN 1 "register_operand" "")
1162 (match_operand:VDN 2 "register_operand" "")]
1163 "TARGET_SIMD"
1164{
1165 rtx tempreg = gen_reg_rtx (<VDBL>mode);
8fcc1c1f
TB		 1166 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1167 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
43e9d192 1168
8fcc1c1f
TB		 1169 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1170 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
43e9d192
IB		 1171 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1172 DONE;
1173})
1174
1175;; For quads.
1176
1177(define_insn "vec_pack_trunc_<mode>"
7973ca62 1178 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
43e9d192
IB		 1179 (vec_concat:<VNARROWQ2>
1180 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1181 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1182 "TARGET_SIMD"
8fcc1c1f
TB		 1183 {
1184 if (BYTES_BIG_ENDIAN)
1185 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1186 else
1187 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1188 }
78ec3036 1189 [(set_attr "type" "multiple")
43e9d192
IB		 1190 (set_attr "length" "8")]
1191)
1192
1193;; Widening operations.
1194
1195(define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1196 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1197 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1198 (match_operand:VQW 1 "register_operand" "w")
1199 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1200 )))]
1201 "TARGET_SIMD"
130ee2eb 1202 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
78ec3036 1203 [(set_attr "type" "neon_shift_imm_long")]
43e9d192
IB		 1204)
1205
1206(define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1207 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1208 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1209 (match_operand:VQW 1 "register_operand" "w")
1210 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1211 )))]
1212 "TARGET_SIMD"
130ee2eb 1213 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
78ec3036 1214 [(set_attr "type" "neon_shift_imm_long")]
43e9d192
IB		 1215)
1216
1217(define_expand "vec_unpack<su>_hi_<mode>"
1218 [(match_operand:<VWIDE> 0 "register_operand" "")
1219 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1220 "TARGET_SIMD"
1221 {
1222 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1223 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1224 operands[1], p));
1225 DONE;
1226 }
1227)
1228
1229(define_expand "vec_unpack<su>_lo_<mode>"
1230 [(match_operand:<VWIDE> 0 "register_operand" "")
1231 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1232 "TARGET_SIMD"
1233 {
1234 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1235 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1236 operands[1], p));
1237 DONE;
1238 }
1239)
1240
1241;; Widening arithmetic.
1242
24244e4d
VP		 1243(define_insn "*aarch64_<su>mlal_lo<mode>"
1244 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1245 (plus:<VWIDE>
1246 (mult:<VWIDE>
1247 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1248 (match_operand:VQW 2 "register_operand" "w")
1249 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1250 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1251 (match_operand:VQW 4 "register_operand" "w")
1252 (match_dup 3))))
1253 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1254 "TARGET_SIMD"
1255 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
78ec3036 1256 [(set_attr "type" "neon_mla_<Vetype>_long")]
24244e4d
VP		 1257)
1258
1259(define_insn "*aarch64_<su>mlal_hi<mode>"
1260 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1261 (plus:<VWIDE>
1262 (mult:<VWIDE>
1263 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1264 (match_operand:VQW 2 "register_operand" "w")
1265 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1266 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1267 (match_operand:VQW 4 "register_operand" "w")
1268 (match_dup 3))))
1269 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1270 "TARGET_SIMD"
1271 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
78ec3036 1272 [(set_attr "type" "neon_mla_<Vetype>_long")]
24244e4d
VP		 1273)
1274
1275(define_insn "*aarch64_<su>mlsl_lo<mode>"
1276 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1277 (minus:<VWIDE>
1278 (match_operand:<VWIDE> 1 "register_operand" "0")
1279 (mult:<VWIDE>
1280 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1281 (match_operand:VQW 2 "register_operand" "w")
1282 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1283 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1284 (match_operand:VQW 4 "register_operand" "w")
1285 (match_dup 3))))))]
1286 "TARGET_SIMD"
1287 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
78ec3036 1288 [(set_attr "type" "neon_mla_<Vetype>_long")]
24244e4d
VP		 1289)
1290
1291(define_insn "*aarch64_<su>mlsl_hi<mode>"
1292 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1293 (minus:<VWIDE>
1294 (match_operand:<VWIDE> 1 "register_operand" "0")
1295 (mult:<VWIDE>
1296 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1297 (match_operand:VQW 2 "register_operand" "w")
1298 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1299 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1300 (match_operand:VQW 4 "register_operand" "w")
1301 (match_dup 3))))))]
1302 "TARGET_SIMD"
1303 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
78ec3036 1304 [(set_attr "type" "neon_mla_<Vetype>_long")]
24244e4d
VP		 1305)
1306
1307(define_insn "*aarch64_<su>mlal<mode>"
1308 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1309 (plus:<VWIDE>
1310 (mult:<VWIDE>
1311 (ANY_EXTEND:<VWIDE>
a844a695 1312 (match_operand:VD_BHSI 1 "register_operand" "w"))
24244e4d 1313 (ANY_EXTEND:<VWIDE>
a844a695 1314 (match_operand:VD_BHSI 2 "register_operand" "w")))
24244e4d
VP		 1315 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1316 "TARGET_SIMD"
1317 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1318 [(set_attr "type" "neon_mla_<Vetype>_long")]
24244e4d
VP		 1319)
1320
1321(define_insn "*aarch64_<su>mlsl<mode>"
1322 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1323 (minus:<VWIDE>
1324 (match_operand:<VWIDE> 1 "register_operand" "0")
1325 (mult:<VWIDE>
1326 (ANY_EXTEND:<VWIDE>
a844a695 1327 (match_operand:VD_BHSI 2 "register_operand" "w"))
24244e4d 1328 (ANY_EXTEND:<VWIDE>
a844a695 1329 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
24244e4d
VP		 1330 "TARGET_SIMD"
1331 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
78ec3036 1332 [(set_attr "type" "neon_mla_<Vetype>_long")]
24244e4d
VP		 1333)
1334
43e9d192
IB		 1335(define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1336 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1337 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1338 (match_operand:VQW 1 "register_operand" "w")
1339 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1340 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1341 (match_operand:VQW 2 "register_operand" "w")
1342 (match_dup 3)))))]
1343 "TARGET_SIMD"
a02ad1aa 1344 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
78ec3036 1345 [(set_attr "type" "neon_mul_<Vetype>_long")]
43e9d192
IB		 1346)
1347
1348(define_expand "vec_widen_<su>mult_lo_<mode>"
1349 [(match_operand:<VWIDE> 0 "register_operand" "")
1350 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1351 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1352 "TARGET_SIMD"
1353 {
1354 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1355 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1356 operands[1],
1357 operands[2], p));
1358 DONE;
1359 }
1360)
1361
1362(define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1363 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1364 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1365 (match_operand:VQW 1 "register_operand" "w")
1366 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1367 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1368 (match_operand:VQW 2 "register_operand" "w")
1369 (match_dup 3)))))]
1370 "TARGET_SIMD"
a02ad1aa 1371 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1372 [(set_attr "type" "neon_mul_<Vetype>_long")]
43e9d192
IB		 1373)
1374
1375(define_expand "vec_widen_<su>mult_hi_<mode>"
1376 [(match_operand:<VWIDE> 0 "register_operand" "")
1377 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1378 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1379 "TARGET_SIMD"
1380 {
1381 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1382 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1383 operands[1],
1384 operands[2], p));
1385 DONE;
1386
1387 }
1388)
1389
1390;; FP vector operations.
1391;; AArch64 AdvSIMD supports single-precision (32-bit) and
1392;; double-precision (64-bit) floating-point data types and arithmetic as
1393;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1394;; without the need for -ffast-math or -funsafe-math-optimizations.
1395;;
1396;; Floating-point operations can raise an exception. Vectorizing such
1397;; operations are safe because of reasons explained below.
1398;;
1399;; ARMv8 permits an extension to enable trapped floating-point
1400;; exception handling, however this is an optional feature. In the
1401;; event of a floating-point exception being raised by vectorised
1402;; code then:
1403;; 1. If trapped floating-point exceptions are available, then a trap
1404;; will be taken when any lane raises an enabled exception. A trap
1405;; handler may determine which lane raised the exception.
1406;; 2. Alternatively a sticky exception flag is set in the
1407;; floating-point status register (FPSR). Software may explicitly
1408;; test the exception flags, in which case the tests will either
1409;; prevent vectorisation, allowing precise identification of the
1410;; failing operation, or if tested outside of vectorisable regions
1411;; then the specific operation and lane are not of interest.
1412
1413;; FP arithmetic operations.
1414
1415(define_insn "add<mode>3"
1416 [(set (match_operand:VDQF 0 "register_operand" "=w")
1417 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1418 (match_operand:VDQF 2 "register_operand" "w")))]
1419 "TARGET_SIMD"
1420 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1421 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
43e9d192
IB		 1422)
1423
1424(define_insn "sub<mode>3"
1425 [(set (match_operand:VDQF 0 "register_operand" "=w")
1426 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1427 (match_operand:VDQF 2 "register_operand" "w")))]
1428 "TARGET_SIMD"
1429 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1430 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
43e9d192
IB		 1431)
1432
1433(define_insn "mul<mode>3"
1434 [(set (match_operand:VDQF 0 "register_operand" "=w")
1435 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1436 (match_operand:VDQF 2 "register_operand" "w")))]
1437 "TARGET_SIMD"
1438 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1439 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
43e9d192
IB		 1440)
1441
1442(define_insn "div<mode>3"
1443 [(set (match_operand:VDQF 0 "register_operand" "=w")
1444 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1445 (match_operand:VDQF 2 "register_operand" "w")))]
1446 "TARGET_SIMD"
1447 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1448 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
43e9d192
IB		 1449)
1450
1451(define_insn "neg<mode>2"
1452 [(set (match_operand:VDQF 0 "register_operand" "=w")
1453 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1454 "TARGET_SIMD"
1455 "fneg\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 1456 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
43e9d192
IB		 1457)
1458
1459(define_insn "abs<mode>2"
1460 [(set (match_operand:VDQF 0 "register_operand" "=w")
1461 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1462 "TARGET_SIMD"
1463 "fabs\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 1464 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
43e9d192
IB		 1465)
1466
1467(define_insn "fma<mode>4"
1468 [(set (match_operand:VDQF 0 "register_operand" "=w")
1469 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1470 (match_operand:VDQF 2 "register_operand" "w")
1471 (match_operand:VDQF 3 "register_operand" "0")))]
1472 "TARGET_SIMD"
1473 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1474 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
43e9d192
IB		 1475)
1476
828e70c1
JG		 1477(define_insn "*aarch64_fma4_elt<mode>"
1478 [(set (match_operand:VDQF 0 "register_operand" "=w")
1479 (fma:VDQF
1480 (vec_duplicate:VDQF
1481 (vec_select:<VEL>
1482 (match_operand:VDQF 1 "register_operand" "<h_con>")
1483 (parallel [(match_operand:SI 2 "immediate_operand")])))
1484 (match_operand:VDQF 3 "register_operand" "w")
1485 (match_operand:VDQF 4 "register_operand" "0")))]
1486 "TARGET_SIMD"
1dd055a2
JG		 1487 {
1488 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1489 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1490 }
78ec3036 1491 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 1492)
1493
1494(define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1495 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1496 (fma:VDQSF
1497 (vec_duplicate:VDQSF
1498 (vec_select:<VEL>
1499 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1500 (parallel [(match_operand:SI 2 "immediate_operand")])))
1501 (match_operand:VDQSF 3 "register_operand" "w")
1502 (match_operand:VDQSF 4 "register_operand" "0")))]
1503 "TARGET_SIMD"
1dd055a2
JG		 1504 {
1505 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1506 INTVAL (operands[2])));
1507 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1508 }
78ec3036 1509 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 1510)
1511
1512(define_insn "*aarch64_fma4_elt_to_128df"
1513 [(set (match_operand:V2DF 0 "register_operand" "=w")
1514 (fma:V2DF
1515 (vec_duplicate:V2DF
1516 (match_operand:DF 1 "register_operand" "w"))
1517 (match_operand:V2DF 2 "register_operand" "w")
1518 (match_operand:V2DF 3 "register_operand" "0")))]
1519 "TARGET_SIMD"
1520 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
78ec3036 1521 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
828e70c1
JG		 1522)
1523
1524(define_insn "*aarch64_fma4_elt_to_64v2df"
1525 [(set (match_operand:DF 0 "register_operand" "=w")
1526 (fma:DF
1527 (vec_select:DF
1528 (match_operand:V2DF 1 "register_operand" "w")
1529 (parallel [(match_operand:SI 2 "immediate_operand")]))
1530 (match_operand:DF 3 "register_operand" "w")
1531 (match_operand:DF 4 "register_operand" "0")))]
1532 "TARGET_SIMD"
1dd055a2
JG		 1533 {
1534 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1535 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1536 }
78ec3036 1537 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
828e70c1
JG		 1538)
1539
1540(define_insn "fnma<mode>4"
1541 [(set (match_operand:VDQF 0 "register_operand" "=w")
1542 (fma:VDQF
1543 (match_operand:VDQF 1 "register_operand" "w")
1544 (neg:VDQF
1545 (match_operand:VDQF 2 "register_operand" "w"))
1546 (match_operand:VDQF 3 "register_operand" "0")))]
1547 "TARGET_SIMD"
1548 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1549 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
828e70c1
JG		 1550)
1551
1552(define_insn "*aarch64_fnma4_elt<mode>"
1553 [(set (match_operand:VDQF 0 "register_operand" "=w")
1554 (fma:VDQF
1555 (neg:VDQF
1556 (match_operand:VDQF 3 "register_operand" "w"))
1557 (vec_duplicate:VDQF
1558 (vec_select:<VEL>
1559 (match_operand:VDQF 1 "register_operand" "<h_con>")
1560 (parallel [(match_operand:SI 2 "immediate_operand")])))
1561 (match_operand:VDQF 4 "register_operand" "0")))]
1562 "TARGET_SIMD"
1dd055a2
JG		 1563 {
1564 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1565 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1566 }
78ec3036 1567 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 1568)
1569
1570(define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1571 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1572 (fma:VDQSF
1573 (neg:VDQSF
1574 (match_operand:VDQSF 3 "register_operand" "w"))
1575 (vec_duplicate:VDQSF
1576 (vec_select:<VEL>
1577 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1578 (parallel [(match_operand:SI 2 "immediate_operand")])))
1579 (match_operand:VDQSF 4 "register_operand" "0")))]
1580 "TARGET_SIMD"
1dd055a2
JG		 1581 {
1582 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1583 INTVAL (operands[2])));
1584 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1585 }
78ec3036 1586 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
828e70c1
JG		 1587)
1588
1589(define_insn "*aarch64_fnma4_elt_to_128df"
1590 [(set (match_operand:V2DF 0 "register_operand" "=w")
1591 (fma:V2DF
1592 (neg:V2DF
1593 (match_operand:V2DF 2 "register_operand" "w"))
1594 (vec_duplicate:V2DF
1595 (match_operand:DF 1 "register_operand" "w"))
1596 (match_operand:V2DF 3 "register_operand" "0")))]
1597 "TARGET_SIMD"
1598 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
78ec3036 1599 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
828e70c1
JG		 1600)
1601
1602(define_insn "*aarch64_fnma4_elt_to_64v2df"
1603 [(set (match_operand:DF 0 "register_operand" "=w")
1604 (fma:DF
1605 (vec_select:DF
1606 (match_operand:V2DF 1 "register_operand" "w")
1607 (parallel [(match_operand:SI 2 "immediate_operand")]))
1608 (neg:DF
1609 (match_operand:DF 3 "register_operand" "w"))
1610 (match_operand:DF 4 "register_operand" "0")))]
1611 "TARGET_SIMD"
1dd055a2
JG		 1612 {
1613 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1614 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1615 }
78ec3036 1616 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
828e70c1
JG		 1617)
1618
0659ce6f 1619;; Vector versions of the floating-point frint patterns.
74dc11ed 1620;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
0659ce6f 1621(define_insn "<frint_pattern><mode>2"
42fc9a7f
JG		 1622 [(set (match_operand:VDQF 0 "register_operand" "=w")
1623 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1624 FRINT))]
1625 "TARGET_SIMD"
1626 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 1627 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
42fc9a7f
JG		 1628)
1629
ce966824
JG		 1630;; Vector versions of the fcvt standard patterns.
1631;; Expands to lbtrunc, lround, lceil, lfloor
1632(define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
42fc9a7f
JG		 1633 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1634 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1635 [(match_operand:VDQF 1 "register_operand" "w")]
1636 FCVT)))]
1637 "TARGET_SIMD"
1638 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 1639 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
42fc9a7f
JG		 1640)
1641
384be29f
JG		 1642(define_expand "<optab><VDQF:mode><fcvt_target>2"
1643 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1644 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1645 [(match_operand:VDQF 1 "register_operand")]
1646 UNSPEC_FRINTZ)))]
1647 "TARGET_SIMD"
1648 {})
1649
1650(define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1651 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1652 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1653 [(match_operand:VDQF 1 "register_operand")]
1654 UNSPEC_FRINTZ)))]
1655 "TARGET_SIMD"
1656 {})
1657
1658(define_expand "ftrunc<VDQF:mode>2"
1659 [(set (match_operand:VDQF 0 "register_operand")
1660 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1661 UNSPEC_FRINTZ))]
1662 "TARGET_SIMD"
1663 {})
1664
1709ff9b
JG		 1665(define_insn "<optab><fcvt_target><VDQF:mode>2"
1666 [(set (match_operand:VDQF 0 "register_operand" "=w")
1667 (FLOATUORS:VDQF
1668 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1669 "TARGET_SIMD"
1670 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 1671 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1709ff9b
JG		 1672)
1673
4c871069
JG		 1674;; Conversions between vectors of floats and doubles.
1675;; Contains a mix of patterns to match standard pattern names
1676;; and those for intrinsics.
1677
1678;; Float widening operations.
1679
1680(define_insn "vec_unpacks_lo_v4sf"
1681 [(set (match_operand:V2DF 0 "register_operand" "=w")
1682 (float_extend:V2DF
1683 (vec_select:V2SF
1684 (match_operand:V4SF 1 "register_operand" "w")
1685 (parallel [(const_int 0) (const_int 1)])
1686 )))]
1687 "TARGET_SIMD"
1688 "fcvtl\\t%0.2d, %1.2s"
78ec3036 1689 [(set_attr "type" "neon_fp_cvt_widen_s")]
4c871069
JG		 1690)
1691
1692(define_insn "aarch64_float_extend_lo_v2df"
1693 [(set (match_operand:V2DF 0 "register_operand" "=w")
1694 (float_extend:V2DF
1695 (match_operand:V2SF 1 "register_operand" "w")))]
1696 "TARGET_SIMD"
1697 "fcvtl\\t%0.2d, %1.2s"
78ec3036 1698 [(set_attr "type" "neon_fp_cvt_widen_s")]
4c871069
JG		 1699)
1700
1701(define_insn "vec_unpacks_hi_v4sf"
1702 [(set (match_operand:V2DF 0 "register_operand" "=w")
1703 (float_extend:V2DF
1704 (vec_select:V2SF
1705 (match_operand:V4SF 1 "register_operand" "w")
1706 (parallel [(const_int 2) (const_int 3)])
1707 )))]
1708 "TARGET_SIMD"
1709 "fcvtl2\\t%0.2d, %1.4s"
78ec3036 1710 [(set_attr "type" "neon_fp_cvt_widen_s")]
4c871069
JG		 1711)
1712
1713;; Float narrowing operations.
1714
1715(define_insn "aarch64_float_truncate_lo_v2sf"
1716 [(set (match_operand:V2SF 0 "register_operand" "=w")
1717 (float_truncate:V2SF
1718 (match_operand:V2DF 1 "register_operand" "w")))]
1719 "TARGET_SIMD"
1720 "fcvtn\\t%0.2s, %1.2d"
78ec3036 1721 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
4c871069
JG		 1722)
1723
1724(define_insn "aarch64_float_truncate_hi_v4sf"
1725 [(set (match_operand:V4SF 0 "register_operand" "=w")
1726 (vec_concat:V4SF
1727 (match_operand:V2SF 1 "register_operand" "0")
1728 (float_truncate:V2SF
1729 (match_operand:V2DF 2 "register_operand" "w"))))]
1730 "TARGET_SIMD"
1731 "fcvtn2\\t%0.4s, %2.2d"
78ec3036 1732 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
4c871069
JG		 1733)
1734
1735(define_expand "vec_pack_trunc_v2df"
1736 [(set (match_operand:V4SF 0 "register_operand")
1737 (vec_concat:V4SF
1738 (float_truncate:V2SF
1739 (match_operand:V2DF 1 "register_operand"))
1740 (float_truncate:V2SF
1741 (match_operand:V2DF 2 "register_operand"))
1742 ))]
1743 "TARGET_SIMD"
1744 {
1745 rtx tmp = gen_reg_rtx (V2SFmode);
8fcc1c1f
TB		 1746 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1747 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1748
1749 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
4c871069 1750 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
8fcc1c1f 1751 tmp, operands[hi]));
4c871069
JG		 1752 DONE;
1753 }
1754)
1755
1756(define_expand "vec_pack_trunc_df"
1757 [(set (match_operand:V2SF 0 "register_operand")
1758 (vec_concat:V2SF
1759 (float_truncate:SF
1760 (match_operand:DF 1 "register_operand"))
1761 (float_truncate:SF
1762 (match_operand:DF 2 "register_operand"))
1763 ))]
1764 "TARGET_SIMD"
1765 {
1766 rtx tmp = gen_reg_rtx (V2SFmode);
8fcc1c1f
TB		 1767 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1768 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1769
1770 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1771 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
4c871069
JG		 1772 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1773 DONE;
1774 }
1775)
1776
43e9d192
IB		 1777(define_insn "aarch64_vmls<mode>"
1778 [(set (match_operand:VDQF 0 "register_operand" "=w")
1779 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1780 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1781 (match_operand:VDQF 3 "register_operand" "w"))))]
1782 "TARGET_SIMD"
1783 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
78ec3036 1784 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
43e9d192
IB		 1785)
1786
1787;; FP Max/Min
1788;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1789;; expression like:
1790;; a = (b < c) ? b : c;
1791;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1792;; either explicitly or indirectly via -ffast-math.
1793;;
1794;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1795;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1796;; operand will be returned when both operands are zero (i.e. they may not
1797;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1798;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1799;; NaNs.
1800
998eaf97 1801(define_insn "<su><maxmin><mode>3"
43e9d192 1802 [(set (match_operand:VDQF 0 "register_operand" "=w")
998eaf97 1803 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
43e9d192
IB		 1804 (match_operand:VDQF 2 "register_operand" "w")))]
1805 "TARGET_SIMD"
998eaf97 1806 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1807 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
43e9d192
IB		 1808)
1809
998eaf97 1810(define_insn "<maxmin_uns><mode>3"
43e9d192 1811 [(set (match_operand:VDQF 0 "register_operand" "=w")
998eaf97
JG		 1812 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1813 (match_operand:VDQF 2 "register_operand" "w")]
1814 FMAXMIN_UNS))]
43e9d192 1815 "TARGET_SIMD"
998eaf97 1816 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 1817 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
43e9d192
IB		 1818)
1819
36054fab 1820;; 'across lanes' add.
43e9d192 1821
f5156c3e
AL		 1822(define_expand "reduc_plus_scal_<mode>"
1823 [(match_operand:<VEL> 0 "register_operand" "=w")
a844a695 1824 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
f5156c3e
AL		 1825 UNSPEC_ADDV)]
1826 "TARGET_SIMD"
1827 {
1828 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1829 rtx scratch = gen_reg_rtx (<MODE>mode);
1830 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1831 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1832 DONE;
1833 }
1834)
1835
1836(define_expand "reduc_plus_scal_<mode>"
1837 [(match_operand:<VEL> 0 "register_operand" "=w")
1838 (match_operand:V2F 1 "register_operand" "w")]
1839 "TARGET_SIMD"
1840 {
1841 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1842 rtx scratch = gen_reg_rtx (<MODE>mode);
1843 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1844 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1845 DONE;
1846 }
1847)
1848
1849(define_insn "aarch64_reduc_plus_internal<mode>"
43e9d192
IB		 1850 [(set (match_operand:VDQV 0 "register_operand" "=w")
1851 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
f5156c3e 1852 UNSPEC_ADDV))]
43e9d192 1853 "TARGET_SIMD"
92835317 1854 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
78ec3036 1855 [(set_attr "type" "neon_reduc_add<q>")]
43e9d192
IB		 1856)
1857
f5156c3e 1858(define_insn "aarch64_reduc_plus_internalv2si"
43e9d192
IB		 1859 [(set (match_operand:V2SI 0 "register_operand" "=w")
1860 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
f5156c3e 1861 UNSPEC_ADDV))]
43e9d192
IB		 1862 "TARGET_SIMD"
1863 "addp\\t%0.2s, %1.2s, %1.2s"
78ec3036 1864 [(set_attr "type" "neon_reduc_add")]
43e9d192
IB		 1865)
1866
f5156c3e 1867(define_insn "aarch64_reduc_plus_internal<mode>"
36054fab
JG		 1868 [(set (match_operand:V2F 0 "register_operand" "=w")
1869 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
92835317 1870 UNSPEC_FADDV))]
43e9d192 1871 "TARGET_SIMD"
36054fab 1872 "faddp\\t%<Vetype>0, %1.<Vtype>"
78ec3036 1873 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
43e9d192
IB		 1874)
1875
36054fab
JG		 1876(define_insn "aarch64_addpv4sf"
1877 [(set (match_operand:V4SF 0 "register_operand" "=w")
1878 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1879 UNSPEC_FADDV))]
43e9d192 1880 "TARGET_SIMD"
36054fab 1881 "faddp\\t%0.4s, %1.4s, %1.4s"
78ec3036 1882 [(set_attr "type" "neon_fp_reduc_add_s_q")]
43e9d192
IB		 1883)
1884
f5156c3e
AL		 1885(define_expand "reduc_plus_scal_v4sf"
1886 [(set (match_operand:SF 0 "register_operand")
36054fab 1887 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
92835317 1888 UNSPEC_FADDV))]
36054fab
JG		 1889 "TARGET_SIMD"
1890{
f5156c3e
AL		 1891 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
1892 rtx scratch = gen_reg_rtx (V4SFmode);
1893 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
1894 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
1895 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
36054fab
JG		 1896 DONE;
1897})
1898
a5e69cad
FY		 1899(define_insn "clrsb<mode>2"
1900 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1901 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1902 "TARGET_SIMD"
1903 "cls\\t%0.<Vtype>, %1.<Vtype>"
1904 [(set_attr "type" "neon_cls<q>")]
1905)
1906
b5574232
VP		 1907(define_insn "clz<mode>2"
1908 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1909 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1910 "TARGET_SIMD"
1911 "clz\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 1912 [(set_attr "type" "neon_cls<q>")]
b5574232
VP		 1913)
1914
a5e69cad
FY		 1915(define_insn "popcount<mode>2"
1916 [(set (match_operand:VB 0 "register_operand" "=w")
1917 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
1918 "TARGET_SIMD"
1919 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
1920 [(set_attr "type" "neon_cnt<q>")]
1921)
1922
998eaf97
JG		 1923;; 'across lanes' max and min ops.
1924
64b0f928
AL		 1925;; Template for outputting a scalar, so we can create __builtins which can be
1926;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
1927(define_expand "reduc_<maxmin_uns>_scal_<mode>"
1928 [(match_operand:<VEL> 0 "register_operand")
1929 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1930 FMAXMINV)]
1931 "TARGET_SIMD"
1932 {
1933 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1934 rtx scratch = gen_reg_rtx (<MODE>mode);
1935 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1936 operands[1]));
1937 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1938 DONE;
1939 }
1940)
1941
1942;; Likewise for integer cases, signed and unsigned.
1943(define_expand "reduc_<maxmin_uns>_scal_<mode>"
1944 [(match_operand:<VEL> 0 "register_operand")
1945 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
1946 MAXMINV)]
1947 "TARGET_SIMD"
1948 {
1949 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1950 rtx scratch = gen_reg_rtx (<MODE>mode);
1951 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1952 operands[1]));
1953 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1954 DONE;
1955 }
1956)
1957
1958(define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
92835317
TB		 1959 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
1960 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
43e9d192
IB		 1961 MAXMINV))]
1962 "TARGET_SIMD"
998eaf97 1963 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
78ec3036 1964 [(set_attr "type" "neon_reduc_minmax<q>")]
43e9d192
IB		 1965)
1966
64b0f928 1967(define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
43e9d192
IB		 1968 [(set (match_operand:V2SI 0 "register_operand" "=w")
1969 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1970 MAXMINV))]
1971 "TARGET_SIMD"
998eaf97 1972 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
78ec3036 1973 [(set_attr "type" "neon_reduc_minmax")]
43e9d192
IB		 1974)
1975
64b0f928
AL		 1976(define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1977 [(set (match_operand:VDQF 0 "register_operand" "=w")
1978 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
998eaf97
JG		 1979 FMAXMINV))]
1980 "TARGET_SIMD"
64b0f928 1981 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
78ec3036 1982 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
998eaf97
JG		 1983)
1984
09962a4a
JG		 1985;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1986;; allocation.
1987;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1988;; to select.
1989;;
1990;; Thus our BSL is of the form:
1991;; op0 = bsl (mask, op2, op3)
4fda1ad1 1992;; We can use any of:
09962a4a
JG		 1993;;
1994;; if (op0 = mask)
1995;; bsl mask, op1, op2
1996;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1997;; bit op0, op2, mask
1998;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1999;; bif op0, op1, mask
4fda1ad1
JG		 2000
2001(define_insn "aarch64_simd_bsl<mode>_internal"
36650ec6
KT		 2002 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2003 (xor:VSDQ_I_DI
c8824f2c 2004 (and:VSDQ_I_DI
36650ec6
KT		 2005 (xor:VSDQ_I_DI
2006 (match_operand:<V_cmp_result> 3 "register_operand" "w,0,w")
2007 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2008 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2009 (match_dup:<V_cmp_result> 3)
09962a4a 2010 ))]
4fda1ad1
JG		 2011 "TARGET_SIMD"
2012 "@
2013 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2014 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2015 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
78ec3036 2016 [(set_attr "type" "neon_bsl<q>")]
4fda1ad1
JG		 2017)
2018
2019(define_expand "aarch64_simd_bsl<mode>"
46e778c4 2020 [(match_operand:VALLDIF 0 "register_operand")
09962a4a 2021 (match_operand:<V_cmp_result> 1 "register_operand")
46e778c4
JG		 2022 (match_operand:VALLDIF 2 "register_operand")
2023 (match_operand:VALLDIF 3 "register_operand")]
09962a4a 2024 "TARGET_SIMD"
4fda1ad1
JG		 2025{
2026 /* We can't alias operands together if they have different modes. */
c8824f2c
JG		 2027 rtx tmp = operands[0];
2028 if (FLOAT_MODE_P (<MODE>mode))
2029 {
2030 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2031 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2032 tmp = gen_reg_rtx (<V_cmp_result>mode);
2033 }
385eb93d 2034 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
c8824f2c
JG		 2035 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2036 operands[1],
2037 operands[2],
2038 operands[3]));
2039 if (tmp != operands[0])
2040 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2041
09962a4a 2042 DONE;
4fda1ad1
JG		 2043})
2044
fc21784d 2045(define_expand "aarch64_vcond_internal<mode><mode>"
a844a695
AL		 2046 [(set (match_operand:VDQ_I 0 "register_operand")
2047 (if_then_else:VDQ_I
4fda1ad1 2048 (match_operator 3 "comparison_operator"
a844a695
AL		 2049 [(match_operand:VDQ_I 4 "register_operand")
2050 (match_operand:VDQ_I 5 "nonmemory_operand")])
2051 (match_operand:VDQ_I 1 "nonmemory_operand")
2052 (match_operand:VDQ_I 2 "nonmemory_operand")))]
4fda1ad1
JG		 2053 "TARGET_SIMD"
2054{
7c19979f
JG		 2055 rtx op1 = operands[1];
2056 rtx op2 = operands[2];
4fda1ad1 2057 rtx mask = gen_reg_rtx (<MODE>mode);
ddeabd3e
AL		 2058 enum rtx_code code = GET_CODE (operands[3]);
2059
2060 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
2061 and desirable for other comparisons if it results in FOO ? -1 : 0
2062 (this allows direct use of the comparison result without a bsl). */
2063 if (code == NE
2064 || (code != EQ
2065 && op1 == CONST0_RTX (<V_cmp_result>mode)
2066 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
2067 {
2068 op1 = operands[2];
2069 op2 = operands[1];
2070 switch (code)
2071 {
2072 case LE: code = GT; break;
2073 case LT: code = GE; break;
2074 case GE: code = LT; break;
2075 case GT: code = LE; break;
2076 /* No case EQ. */
2077 case NE: code = EQ; break;
2078 case LTU: code = GEU; break;
2079 case LEU: code = GTU; break;
2080 case GTU: code = LEU; break;
2081 case GEU: code = LTU; break;
2082 default: gcc_unreachable ();
2083 }
2084 }
4fda1ad1 2085
ddeabd3e
AL		 2086 /* Make sure we can handle the last operand. */
2087 switch (code)
4fda1ad1 2088 {
ddeabd3e
AL		 2089 case NE:
2090 /* Normalized to EQ above. */
2091 gcc_unreachable ();
2092
4fda1ad1
JG		 2093 case LE:
2094 case LT:
4fda1ad1
JG		 2095 case GE:
2096 case GT:
2097 case EQ:
ddeabd3e
AL		 2098 /* These instructions have a form taking an immediate zero. */
2099 if (operands[5] == CONST0_RTX (<MODE>mode))
2100 break;
2101 /* Fall through, as may need to load into register. */
4fda1ad1 2102 default:
ddeabd3e
AL		 2103 if (!REG_P (operands[5]))
2104 operands[5] = force_reg (<MODE>mode, operands[5]);
4fda1ad1
JG		 2105 break;
2106 }
2107
ddeabd3e 2108 switch (code)
4fda1ad1
JG		 2109 {
2110 case LT:
ddeabd3e
AL		 2111 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2112 break;
2113
4fda1ad1
JG		 2114 case GE:
2115 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2116 break;
2117
2118 case LE:
ddeabd3e
AL		 2119 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2120 break;
2121
4fda1ad1
JG		 2122 case GT:
2123 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2124 break;
2125
2126 case LTU:
ddeabd3e
AL		 2127 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2128 break;
2129
4fda1ad1 2130 case GEU:
889b9412 2131 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
4fda1ad1
JG		 2132 break;
2133
2134 case LEU:
ddeabd3e
AL		 2135 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2136 break;
2137
4fda1ad1 2138 case GTU:
889b9412 2139 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
4fda1ad1
JG		 2140 break;
2141
ddeabd3e 2142 /* NE has been normalized to EQ above. */
4fda1ad1
JG		 2143 case EQ:
2144 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2145 break;
2146
2147 default:
2148 gcc_unreachable ();
2149 }
2150
7c19979f
JG		 2151 /* If we have (a = (b CMP c) ? -1 : 0);
2152 Then we can simply move the generated mask. */
2153
2154 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2155 && op2 == CONST0_RTX (<V_cmp_result>mode))
2156 emit_move_insn (operands[0], mask);
2157 else
2158 {
2159 if (!REG_P (op1))
2160 op1 = force_reg (<MODE>mode, op1);
2161 if (!REG_P (op2))
2162 op2 = force_reg (<MODE>mode, op2);
2163 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2164 op1, op2));
2165 }
4fda1ad1
JG		 2166
2167 DONE;
2168})
2169
fc21784d
JG		 2170(define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2171 [(set (match_operand:VDQF_COND 0 "register_operand")
385eb93d
JG		 2172 (if_then_else:VDQF
2173 (match_operator 3 "comparison_operator"
2174 [(match_operand:VDQF 4 "register_operand")
2175 (match_operand:VDQF 5 "nonmemory_operand")])
fc21784d
JG		 2176 (match_operand:VDQF_COND 1 "nonmemory_operand")
2177 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
385eb93d
JG		 2178 "TARGET_SIMD"
2179{
2180 int inverse = 0;
d07458be 2181 int use_zero_form = 0;
70c67693 2182 int swap_bsl_operands = 0;
7c19979f
JG		 2183 rtx op1 = operands[1];
2184 rtx op2 = operands[2];
fc21784d
JG		 2185 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2186 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
385eb93d 2187
70c67693
JG		 2188 rtx (*base_comparison) (rtx, rtx, rtx);
2189 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2190
2191 switch (GET_CODE (operands[3]))
2192 {
2193 case GE:
d07458be 2194 case GT:
70c67693 2195 case LE:
d07458be 2196 case LT:
70c67693 2197 case EQ:
d07458be
JG		 2198 if (operands[5] == CONST0_RTX (<MODE>mode))
2199 {
2200 use_zero_form = 1;
2201 break;
2202 }
2203 /* Fall through. */
70c67693
JG		 2204 default:
2205 if (!REG_P (operands[5]))
fc21784d 2206 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
70c67693 2207 }
385eb93d
JG		 2208
2209 switch (GET_CODE (operands[3]))
2210 {
2211 case LT:
70c67693 2212 case UNLT:
385eb93d
JG		 2213 inverse = 1;
2214 /* Fall through. */
2215 case GE:
70c67693
JG		 2216 case UNGE:
2217 case ORDERED:
2218 case UNORDERED:
fc21784d
JG		 2219 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2220 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
385eb93d
JG		 2221 break;
2222 case LE:
70c67693 2223 case UNLE:
385eb93d
JG		 2224 inverse = 1;
2225 /* Fall through. */
2226 case GT:
70c67693 2227 case UNGT:
fc21784d
JG		 2228 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2229 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
385eb93d 2230 break;
70c67693 2231 case EQ:
385eb93d 2232 case NE:
70c67693 2233 case UNEQ:
fc21784d
JG		 2234 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2235 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
70c67693
JG		 2236 break;
2237 default:
2238 gcc_unreachable ();
2239 }
2240
2241 switch (GET_CODE (operands[3]))
2242 {
2243 case LT:
2244 case LE:
2245 case GT:
2246 case GE:
385eb93d 2247 case EQ:
70c67693
JG		 2248 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2249 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2250 a GE b -> a GE b
2251 a GT b -> a GT b
2252 a LE b -> b GE a
2253 a LT b -> b GT a
d07458be
JG		 2254 a EQ b -> a EQ b
2255 Note that there also exist direct comparison against 0 forms,
2256 so catch those as a special case. */
2257 if (use_zero_form)
2258 {
2259 inverse = 0;
2260 switch (GET_CODE (operands[3]))
2261 {
2262 case LT:
fc21784d 2263 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
d07458be
JG		 2264 break;
2265 case LE:
fc21784d 2266 base_comparison = gen_aarch64_cmle<VDQF:mode>;
d07458be
JG		 2267 break;
2268 default:
2269 /* Do nothing, other zero form cases already have the correct
2270 base_comparison. */
2271 break;
2272 }
2273 }
70c67693
JG		 2274
2275 if (!inverse)
2276 emit_insn (base_comparison (mask, operands[4], operands[5]));
2277 else
2278 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2279 break;
2280 case UNLT:
2281 case UNLE:
2282 case UNGT:
2283 case UNGE:
2284 case NE:
2285 /* FCM returns false for lanes which are unordered, so if we use
2286 the inverse of the comparison we actually want to emit, then
2287 swap the operands to BSL, we will end up with the correct result.
2288 Note that a NE NaN and NaN NE b are true for all a, b.
2289
2290 Our transformations are:
2291 a GE b -> !(b GT a)
2292 a GT b -> !(b GE a)
2293 a LE b -> !(a GT b)
2294 a LT b -> !(a GE b)
2295 a NE b -> !(a EQ b) */
2296
2297 if (inverse)
2298 emit_insn (base_comparison (mask, operands[4], operands[5]));
2299 else
2300 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2301
2302 swap_bsl_operands = 1;
2303 break;
2304 case UNEQ:
2305 /* We check (a > b || b > a). combining these comparisons give us
2306 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2307 will then give us (a == b || a UNORDERED b) as intended. */
2308
fc21784d
JG		 2309 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2310 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2311 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
70c67693
JG		 2312 swap_bsl_operands = 1;
2313 break;
2314 case UNORDERED:
2315 /* Operands are ORDERED iff (a > b || b >= a).
2316 Swapping the operands to BSL will give the UNORDERED case. */
2317 swap_bsl_operands = 1;
2318 /* Fall through. */
2319 case ORDERED:
fc21784d
JG		 2320 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2321 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2322 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
385eb93d
JG		 2323 break;
2324 default:
2325 gcc_unreachable ();
2326 }
2327
70c67693 2328 if (swap_bsl_operands)
7c19979f
JG		 2329 {
2330 op1 = operands[2];
2331 op2 = operands[1];
2332 }
2333
2334 /* If we have (a = (b CMP c) ? -1 : 0);
2335 Then we can simply move the generated mask. */
2336
fc21784d
JG		 2337 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2338 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
7c19979f
JG		 2339 emit_move_insn (operands[0], mask);
2340 else
2341 {
2342 if (!REG_P (op1))
fc21784d 2343 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
7c19979f 2344 if (!REG_P (op2))
fc21784d
JG		 2345 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2346 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
7c19979f
JG		 2347 op1, op2));
2348 }
2349
385eb93d
JG		 2350 DONE;
2351})
2352
4fda1ad1 2353(define_expand "vcond<mode><mode>"
385eb93d
JG		 2354 [(set (match_operand:VALL 0 "register_operand")
2355 (if_then_else:VALL
4fda1ad1 2356 (match_operator 3 "comparison_operator"
385eb93d
JG		 2357 [(match_operand:VALL 4 "register_operand")
2358 (match_operand:VALL 5 "nonmemory_operand")])
7c19979f
JG		 2359 (match_operand:VALL 1 "nonmemory_operand")
2360 (match_operand:VALL 2 "nonmemory_operand")))]
4fda1ad1
JG		 2361 "TARGET_SIMD"
2362{
fc21784d 2363 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
4fda1ad1
JG		 2364 operands[2], operands[3],
2365 operands[4], operands[5]));
2366 DONE;
2367})
2368
7c19979f
JG		 2369(define_expand "vcond<v_cmp_result><mode>"
2370 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2371 (if_then_else:<V_cmp_result>
2372 (match_operator 3 "comparison_operator"
2373 [(match_operand:VDQF 4 "register_operand")
2374 (match_operand:VDQF 5 "nonmemory_operand")])
2375 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2376 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2377 "TARGET_SIMD"
2378{
fc21784d 2379 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
7c19979f
JG		 2380 operands[0], operands[1],
2381 operands[2], operands[3],
2382 operands[4], operands[5]));
2383 DONE;
2384})
4fda1ad1
JG		 2385
2386(define_expand "vcondu<mode><mode>"
a844a695
AL		 2387 [(set (match_operand:VDQ_I 0 "register_operand")
2388 (if_then_else:VDQ_I
4fda1ad1 2389 (match_operator 3 "comparison_operator"
a844a695
AL		 2390 [(match_operand:VDQ_I 4 "register_operand")
2391 (match_operand:VDQ_I 5 "nonmemory_operand")])
2392 (match_operand:VDQ_I 1 "nonmemory_operand")
2393 (match_operand:VDQ_I 2 "nonmemory_operand")))]
4fda1ad1
JG		 2394 "TARGET_SIMD"
2395{
fc21784d 2396 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
4fda1ad1
JG		 2397 operands[2], operands[3],
2398 operands[4], operands[5]));
2399 DONE;
2400})
2401
43e9d192
IB		 2402;; Patterns for AArch64 SIMD Intrinsics.
2403
66adb8eb
JG		 2404;; Lane extraction with sign extension to general purpose register.
2405(define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2406 [(set (match_operand:GPI 0 "register_operand" "=r")
2407 (sign_extend:GPI
43e9d192 2408 (vec_select:<VEL>
66adb8eb 2409 (match_operand:VDQQH 1 "register_operand" "w")
43e9d192
IB		 2410 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2411 "TARGET_SIMD"
e58bf20a
TB		 2412 {
2413 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2414 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2415 }
78ec3036 2416 [(set_attr "type" "neon_to_gp<q>")]
43e9d192
IB		 2417)
2418
66adb8eb
JG		 2419(define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2420 [(set (match_operand:SI 0 "register_operand" "=r")
2421 (zero_extend:SI
43e9d192 2422 (vec_select:<VEL>
66adb8eb 2423 (match_operand:VDQQH 1 "register_operand" "w")
43e9d192
IB		 2424 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2425 "TARGET_SIMD"
e58bf20a
TB		 2426 {
2427 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2428 return "umov\\t%w0, %1.<Vetype>[%2]";
2429 }
78ec3036 2430 [(set_attr "type" "neon_to_gp<q>")]
43e9d192
IB		 2431)
2432
66adb8eb
JG		 2433;; Lane extraction of a value, neither sign nor zero extension
2434;; is guaranteed so upper bits should be considered undefined.
ed225d0c 2435;; RTL uses GCC vector extension indices throughout so flip only for assembly.
43e9d192 2436(define_insn "aarch64_get_lane<mode>"
e58bf20a 2437 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
43e9d192 2438 (vec_select:<VEL>
e58bf20a
TB		 2439 (match_operand:VALL 1 "register_operand" "w, w, w")
2440 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
43e9d192 2441 "TARGET_SIMD"
e58bf20a
TB		 2442 {
2443 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2444 switch (which_alternative)
2445 {
2446 case 0:
2447 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2448 case 1:
2449 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2450 case 2:
2451 return "st1\\t{%1.<Vetype>}[%2], %0";
2452 default:
2453 gcc_unreachable ();
2454 }
2455 }
2456 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
43e9d192
IB		 2457)
2458
43e9d192
IB		 2459;; In this insn, operand 1 should be low, and operand 2 the high part of the
2460;; dest vector.
2461
2462(define_insn "*aarch64_combinez<mode>"
2463 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2464 (vec_concat:<VDBL>
a844a695
AL		 2465 (match_operand:VD_BHSI 1 "register_operand" "w")
2466 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz")))]
5a908485
JG		 2467 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2468 "mov\\t%0.8b, %1.8b"
2469 [(set_attr "type" "neon_move<q>")]
2470)
2471
2472(define_insn "*aarch64_combinez_be<mode>"
2473 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2474 (vec_concat:<VDBL>
a844a695
AL		 2475 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz")
2476 (match_operand:VD_BHSI 1 "register_operand" "w")))]
5a908485 2477 "TARGET_SIMD && BYTES_BIG_ENDIAN"
43e9d192 2478 "mov\\t%0.8b, %1.8b"
78ec3036 2479 [(set_attr "type" "neon_move<q>")]
43e9d192
IB		 2480)
2481
5a908485
JG		 2482(define_expand "aarch64_combine<mode>"
2483 [(match_operand:<VDBL> 0 "register_operand")
2484 (match_operand:VDC 1 "register_operand")
2485 (match_operand:VDC 2 "register_operand")]
2486 "TARGET_SIMD"
2487{
2488 rtx op1, op2;
2489 if (BYTES_BIG_ENDIAN)
2490 {
2491 op1 = operands[2];
2492 op2 = operands[1];
2493 }
2494 else
2495 {
2496 op1 = operands[1];
2497 op2 = operands[2];
2498 }
2499 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2500 DONE;
2501}
2502)
2503
2504(define_insn_and_split "aarch64_combine_internal<mode>"
43e9d192
IB		 2505 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2506 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2507 (match_operand:VDC 2 "register_operand" "w")))]
2508 "TARGET_SIMD"
8b033a8a
SN		 2509 "#"
2510 "&& reload_completed"
2511 [(const_int 0)]
2512{
5a908485
JG		 2513 if (BYTES_BIG_ENDIAN)
2514 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2515 else
2516 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
8b033a8a 2517 DONE;
0f686aa9
JG		 2518}
2519[(set_attr "type" "multiple")]
2520)
8b033a8a
SN		 2521
2522(define_expand "aarch64_simd_combine<mode>"
5a908485
JG		 2523 [(match_operand:<VDBL> 0 "register_operand")
2524 (match_operand:VDC 1 "register_operand")
2525 (match_operand:VDC 2 "register_operand")]
8b033a8a
SN		 2526 "TARGET_SIMD"
2527 {
2528 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2529 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2530 DONE;
0f686aa9
JG		 2531 }
2532[(set_attr "type" "multiple")]
2533)
43e9d192
IB		 2534
2535;; <su><addsub>l<q>.
2536
8da00d65 2537(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
43e9d192
IB		 2538 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2539 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2540 (match_operand:VQW 1 "register_operand" "w")
2541 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2542 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2543 (match_operand:VQW 2 "register_operand" "w")
2544 (match_dup 3)))))]
2545 "TARGET_SIMD"
8da00d65 2546 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 2547 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
43e9d192
IB		 2548)
2549
8da00d65
VP		 2550(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2551 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2552 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2553 (match_operand:VQW 1 "register_operand" "w")
2554 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2555 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2556 (match_operand:VQW 2 "register_operand" "w")
2557 (match_dup 3)))))]
2558 "TARGET_SIMD"
2559 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
78ec3036 2560 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
8da00d65
VP		 2561)
2562
2563
43e9d192
IB		 2564(define_expand "aarch64_saddl2<mode>"
2565 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2566 (match_operand:VQW 1 "register_operand" "w")
2567 (match_operand:VQW 2 "register_operand" "w")]
2568 "TARGET_SIMD"
2569{
2570 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
8da00d65
VP		 2571 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2572 operands[2], p));
43e9d192
IB		 2573 DONE;
2574})
2575
2576(define_expand "aarch64_uaddl2<mode>"
2577 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2578 (match_operand:VQW 1 "register_operand" "w")
2579 (match_operand:VQW 2 "register_operand" "w")]
2580 "TARGET_SIMD"
2581{
2582 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
8da00d65
VP		 2583 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2584 operands[2], p));
43e9d192
IB		 2585 DONE;
2586})
2587
2588(define_expand "aarch64_ssubl2<mode>"
2589 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2590 (match_operand:VQW 1 "register_operand" "w")
2591 (match_operand:VQW 2 "register_operand" "w")]
2592 "TARGET_SIMD"
2593{
2594 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
8da00d65 2595 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
43e9d192
IB		 2596 operands[2], p));
2597 DONE;
2598})
2599
2600(define_expand "aarch64_usubl2<mode>"
2601 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2602 (match_operand:VQW 1 "register_operand" "w")
2603 (match_operand:VQW 2 "register_operand" "w")]
2604 "TARGET_SIMD"
2605{
2606 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
8da00d65 2607 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
43e9d192
IB		 2608 operands[2], p));
2609 DONE;
2610})
2611
2612(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2613 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2614 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
a844a695 2615 (match_operand:VD_BHSI 1 "register_operand" "w"))
43e9d192 2616 (ANY_EXTEND:<VWIDE>
a844a695 2617 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
43e9d192 2618 "TARGET_SIMD"
130ee2eb 2619 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 2620 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
43e9d192
IB		 2621)
2622
2623;; <su><addsub>w<q>.
2624
2625(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2626 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2627 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2628 (ANY_EXTEND:<VWIDE>
a844a695 2629 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
43e9d192
IB		 2630 "TARGET_SIMD"
2631 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
78ec3036 2632 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
43e9d192
IB		 2633)
2634
2635(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2636 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2637 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2638 (ANY_EXTEND:<VWIDE>
2639 (vec_select:<VHALF>
2640 (match_operand:VQW 2 "register_operand" "w")
2641 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2642 "TARGET_SIMD"
2643 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
78ec3036 2644 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
43e9d192
IB		 2645)
2646
2647(define_expand "aarch64_saddw2<mode>"
2648 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2649 (match_operand:<VWIDE> 1 "register_operand" "w")
2650 (match_operand:VQW 2 "register_operand" "w")]
2651 "TARGET_SIMD"
2652{
2653 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2654 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2655 operands[2], p));
2656 DONE;
2657})
2658
2659(define_expand "aarch64_uaddw2<mode>"
2660 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2661 (match_operand:<VWIDE> 1 "register_operand" "w")
2662 (match_operand:VQW 2 "register_operand" "w")]
2663 "TARGET_SIMD"
2664{
2665 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2666 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2667 operands[2], p));
2668 DONE;
2669})
2670
2671
2672(define_expand "aarch64_ssubw2<mode>"
2673 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2674 (match_operand:<VWIDE> 1 "register_operand" "w")
2675 (match_operand:VQW 2 "register_operand" "w")]
2676 "TARGET_SIMD"
2677{
2678 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2679 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2680 operands[2], p));
2681 DONE;
2682})
2683
2684(define_expand "aarch64_usubw2<mode>"
2685 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2686 (match_operand:<VWIDE> 1 "register_operand" "w")
2687 (match_operand:VQW 2 "register_operand" "w")]
2688 "TARGET_SIMD"
2689{
2690 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2691 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2692 operands[2], p));
2693 DONE;
2694})
2695
2696;; <su><r>h<addsub>.
2697
2698(define_insn "aarch64_<sur>h<addsub><mode>"
a844a695
AL		 2699 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2700 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
2701 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
43e9d192
IB		 2702 HADDSUB))]
2703 "TARGET_SIMD"
2704 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 2705 [(set_attr "type" "neon_<addsub>_halve<q>")]
43e9d192
IB		 2706)
2707
2708;; <r><addsub>hn<q>.
2709
2710(define_insn "aarch64_<sur><addsub>hn<mode>"
2711 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2712 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2713 (match_operand:VQN 2 "register_operand" "w")]
2714 ADDSUBHN))]
2715 "TARGET_SIMD"
2716 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 2717 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
43e9d192
IB		 2718)
2719
2720(define_insn "aarch64_<sur><addsub>hn2<mode>"
2721 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2722 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2723 (match_operand:VQN 2 "register_operand" "w")
2724 (match_operand:VQN 3 "register_operand" "w")]
2725 ADDSUBHN2))]
2726 "TARGET_SIMD"
2727 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
78ec3036 2728 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
43e9d192
IB		 2729)
2730
2731;; pmul.
2732
2733(define_insn "aarch64_pmul<mode>"
2734 [(set (match_operand:VB 0 "register_operand" "=w")
2735 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2736 (match_operand:VB 2 "register_operand" "w")]
2737 UNSPEC_PMUL))]
2738 "TARGET_SIMD"
2739 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 2740 [(set_attr "type" "neon_mul_<Vetype><q>")]
43e9d192
IB		 2741)
2742
2743;; <su>q<addsub>
2744
2745(define_insn "aarch64_<su_optab><optab><mode>"
2746 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2747 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2748 (match_operand:VSDQ_I 2 "register_operand" "w")))]
2749 "TARGET_SIMD"
2750 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 2751 [(set_attr "type" "neon_<optab><q>")]
43e9d192
IB		 2752)
2753
2754;; suqadd and usqadd
2755
2756(define_insn "aarch64_<sur>qadd<mode>"
2757 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2758 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2759 (match_operand:VSDQ_I 2 "register_operand" "w")]
2760 USSUQADD))]
2761 "TARGET_SIMD"
2762 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
78ec3036 2763 [(set_attr "type" "neon_qadd<q>")]
43e9d192
IB		 2764)
2765
2766;; sqmovun
2767
2768(define_insn "aarch64_sqmovun<mode>"
2769 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2770 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2771 UNSPEC_SQXTUN))]
2772 "TARGET_SIMD"
2773 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
78ec3036 2774 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
43e9d192
IB		 2775 )
2776
2777;; sqmovn and uqmovn
2778
2779(define_insn "aarch64_<sur>qmovn<mode>"
2780 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2781 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2782 SUQMOVN))]
2783 "TARGET_SIMD"
2784 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
78ec3036 2785 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
43e9d192
IB		 2786 )
2787
2788;; <su>q<absneg>
2789
2790(define_insn "aarch64_s<optab><mode>"
9551c7ec
AV		 2791 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2792 (UNQOPS:VSDQ_I
2793 (match_operand:VSDQ_I 1 "register_operand" "w")))]
43e9d192
IB		 2794 "TARGET_SIMD"
2795 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
78ec3036 2796 [(set_attr "type" "neon_<optab><q>")]
43e9d192
IB		 2797)
2798
2799;; sq<r>dmulh.
2800
2801(define_insn "aarch64_sq<r>dmulh<mode>"
2802 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2803 (unspec:VSDQ_HSI
2804 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2805 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2806 VQDMULH))]
2807 "TARGET_SIMD"
2808 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 2809 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
43e9d192
IB		 2810)
2811
2812;; sq<r>dmulh_lane
2813
2a49c16d 2814(define_insn "aarch64_sq<r>dmulh_lane<mode>"
b7d7d917
TB		 2815 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2816 (unspec:VDQHS
2817 [(match_operand:VDQHS 1 "register_operand" "w")
2818 (vec_select:<VEL>
2819 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2820 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2821 VQDMULH))]
2822 "TARGET_SIMD"
2823 "*
1dd055a2 2824 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
b7d7d917 2825 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
78ec3036 2826 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
b7d7d917
TB		 2827)
2828
2a49c16d 2829(define_insn "aarch64_sq<r>dmulh_laneq<mode>"
b7d7d917
TB		 2830 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2831 (unspec:VDQHS
2832 [(match_operand:VDQHS 1 "register_operand" "w")
2833 (vec_select:<VEL>
2834 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2835 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2836 VQDMULH))]
2837 "TARGET_SIMD"
2838 "*
1dd055a2 2839 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
b7d7d917 2840 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
78ec3036 2841 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
b7d7d917
TB		 2842)
2843
2a49c16d 2844(define_insn "aarch64_sq<r>dmulh_lane<mode>"
b7d7d917
TB		 2845 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2846 (unspec:SD_HSI
2847 [(match_operand:SD_HSI 1 "register_operand" "w")
43e9d192 2848 (vec_select:<VEL>
278821f2 2849 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
43e9d192
IB		 2850 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2851 VQDMULH))]
2852 "TARGET_SIMD"
2853 "*
278821f2 2854 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
b7d7d917 2855 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
78ec3036 2856 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
43e9d192
IB		 2857)
2858
2a49c16d 2859(define_insn "aarch64_sq<r>dmulh_laneq<mode>"
d2937a2e
KT		 2860 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2861 (unspec:SD_HSI
2862 [(match_operand:SD_HSI 1 "register_operand" "w")
2863 (vec_select:<VEL>
2864 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2865 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2866 VQDMULH))]
2867 "TARGET_SIMD"
2868 "*
2869 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2870 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2871 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2872)
2873
43e9d192
IB		 2874;; vqdml[sa]l
2875
2876(define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2877 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2878 (SBINQOPS:<VWIDE>
2879 (match_operand:<VWIDE> 1 "register_operand" "0")
2880 (ss_ashift:<VWIDE>
2881 (mult:<VWIDE>
2882 (sign_extend:<VWIDE>
2883 (match_operand:VSD_HSI 2 "register_operand" "w"))
2884 (sign_extend:<VWIDE>
2885 (match_operand:VSD_HSI 3 "register_operand" "w")))
2886 (const_int 1))))]
2887 "TARGET_SIMD"
2888 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
78ec3036 2889 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
43e9d192
IB		 2890)
2891
2892;; vqdml[sa]l_lane
2893
2a49c16d 2894(define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
43e9d192
IB		 2895 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2896 (SBINQOPS:<VWIDE>
2897 (match_operand:<VWIDE> 1 "register_operand" "0")
2898 (ss_ashift:<VWIDE>
2899 (mult:<VWIDE>
2900 (sign_extend:<VWIDE>
2901 (match_operand:VD_HSI 2 "register_operand" "w"))
2902 (sign_extend:<VWIDE>
2903 (vec_duplicate:VD_HSI
2904 (vec_select:<VEL>
278821f2
KT		 2905 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2906 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2907 ))
2908 (const_int 1))))]
2909 "TARGET_SIMD"
2910 {
2911 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2912 return
2913 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2914 }
2915 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2916)
2917
2a49c16d 2918(define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
278821f2
KT		 2919 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2920 (SBINQOPS:<VWIDE>
2921 (match_operand:<VWIDE> 1 "register_operand" "0")
2922 (ss_ashift:<VWIDE>
2923 (mult:<VWIDE>
2924 (sign_extend:<VWIDE>
2925 (match_operand:VD_HSI 2 "register_operand" "w"))
2926 (sign_extend:<VWIDE>
2927 (vec_duplicate:VD_HSI
2928 (vec_select:<VEL>
2929 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
43e9d192
IB		 2930 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2931 ))
2932 (const_int 1))))]
2933 "TARGET_SIMD"
1dd055a2
JG		 2934 {
2935 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2936 return
2937 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2938 }
78ec3036 2939 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
43e9d192
IB		 2940)
2941
2a49c16d 2942(define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
43e9d192
IB		 2943 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2944 (SBINQOPS:<VWIDE>
2945 (match_operand:<VWIDE> 1 "register_operand" "0")
2946 (ss_ashift:<VWIDE>
2947 (mult:<VWIDE>
2948 (sign_extend:<VWIDE>
2949 (match_operand:SD_HSI 2 "register_operand" "w"))
2950 (sign_extend:<VWIDE>
2951 (vec_select:<VEL>
278821f2
KT		 2952 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2953 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2954 )
2955 (const_int 1))))]
2956 "TARGET_SIMD"
2957 {
2958 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2959 return
2960 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2961 }
2962 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2963)
2964
2a49c16d 2965(define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
278821f2
KT		 2966 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2967 (SBINQOPS:<VWIDE>
2968 (match_operand:<VWIDE> 1 "register_operand" "0")
2969 (ss_ashift:<VWIDE>
2970 (mult:<VWIDE>
2971 (sign_extend:<VWIDE>
2972 (match_operand:SD_HSI 2 "register_operand" "w"))
2973 (sign_extend:<VWIDE>
2974 (vec_select:<VEL>
2975 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
43e9d192
IB		 2976 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2977 )
2978 (const_int 1))))]
2979 "TARGET_SIMD"
1dd055a2
JG		 2980 {
2981 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2982 return
2983 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2984 }
78ec3036 2985 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
43e9d192
IB		 2986)
2987
43e9d192
IB		 2988;; vqdml[sa]l_n
2989
2990(define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2991 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2992 (SBINQOPS:<VWIDE>
2993 (match_operand:<VWIDE> 1 "register_operand" "0")
2994 (ss_ashift:<VWIDE>
2995 (mult:<VWIDE>
2996 (sign_extend:<VWIDE>
2997 (match_operand:VD_HSI 2 "register_operand" "w"))
2998 (sign_extend:<VWIDE>
2999 (vec_duplicate:VD_HSI
1c83b673 3000 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
43e9d192
IB		 3001 (const_int 1))))]
3002 "TARGET_SIMD"
3003 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
78ec3036 3004 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
43e9d192
IB		 3005)
3006
3007;; sqdml[as]l2
3008
3009(define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3010 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3011 (SBINQOPS:<VWIDE>
3012 (match_operand:<VWIDE> 1 "register_operand" "0")
3013 (ss_ashift:<VWIDE>
3014 (mult:<VWIDE>
3015 (sign_extend:<VWIDE>
3016 (vec_select:<VHALF>
3017 (match_operand:VQ_HSI 2 "register_operand" "w")
3018 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3019 (sign_extend:<VWIDE>
3020 (vec_select:<VHALF>
3021 (match_operand:VQ_HSI 3 "register_operand" "w")
3022 (match_dup 4))))
3023 (const_int 1))))]
3024 "TARGET_SIMD"
3025 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
78ec3036 3026 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
43e9d192
IB		 3027)
3028
3029(define_expand "aarch64_sqdmlal2<mode>"
3030 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3031 (match_operand:<VWIDE> 1 "register_operand" "w")
3032 (match_operand:VQ_HSI 2 "register_operand" "w")
3033 (match_operand:VQ_HSI 3 "register_operand" "w")]
3034 "TARGET_SIMD"
3035{
3036 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3037 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3038 operands[2], operands[3], p));
3039 DONE;
3040})
3041
3042(define_expand "aarch64_sqdmlsl2<mode>"
3043 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3044 (match_operand:<VWIDE> 1 "register_operand" "w")
3045 (match_operand:VQ_HSI 2 "register_operand" "w")
3046 (match_operand:VQ_HSI 3 "register_operand" "w")]
3047 "TARGET_SIMD"
3048{
3049 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3050 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3051 operands[2], operands[3], p));
3052 DONE;
3053})
3054
3055;; vqdml[sa]l2_lane
3056
3057(define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3058 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3059 (SBINQOPS:<VWIDE>
3060 (match_operand:<VWIDE> 1 "register_operand" "0")
3061 (ss_ashift:<VWIDE>
3062 (mult:<VWIDE>
3063 (sign_extend:<VWIDE>
3064 (vec_select:<VHALF>
3065 (match_operand:VQ_HSI 2 "register_operand" "w")
3066 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3067 (sign_extend:<VWIDE>
3068 (vec_duplicate:<VHALF>
3069 (vec_select:<VEL>
278821f2
KT		 3070 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3071 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3072 ))))
3073 (const_int 1))))]
3074 "TARGET_SIMD"
3075 {
3076 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3077 return
3078 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3079 }
3080 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3081)
3082
3083(define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3084 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3085 (SBINQOPS:<VWIDE>
3086 (match_operand:<VWIDE> 1 "register_operand" "0")
3087 (ss_ashift:<VWIDE>
3088 (mult:<VWIDE>
3089 (sign_extend:<VWIDE>
3090 (vec_select:<VHALF>
3091 (match_operand:VQ_HSI 2 "register_operand" "w")
3092 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3093 (sign_extend:<VWIDE>
3094 (vec_duplicate:<VHALF>
3095 (vec_select:<VEL>
3096 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
43e9d192
IB		 3097 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3098 ))))
3099 (const_int 1))))]
3100 "TARGET_SIMD"
1dd055a2
JG		 3101 {
3102 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3103 return
3104 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3105 }
78ec3036 3106 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
43e9d192
IB		 3107)
3108
3109(define_expand "aarch64_sqdmlal2_lane<mode>"
3110 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3111 (match_operand:<VWIDE> 1 "register_operand" "w")
3112 (match_operand:VQ_HSI 2 "register_operand" "w")
278821f2 3113 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
43e9d192
IB		 3114 (match_operand:SI 4 "immediate_operand" "i")]
3115 "TARGET_SIMD"
3116{
3117 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
43e9d192
IB		 3118 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3119 operands[2], operands[3],
3120 operands[4], p));
3121 DONE;
3122})
3123
3124(define_expand "aarch64_sqdmlal2_laneq<mode>"
3125 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3126 (match_operand:<VWIDE> 1 "register_operand" "w")
3127 (match_operand:VQ_HSI 2 "register_operand" "w")
278821f2 3128 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
43e9d192
IB		 3129 (match_operand:SI 4 "immediate_operand" "i")]
3130 "TARGET_SIMD"
3131{
3132 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
278821f2 3133 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
43e9d192
IB		 3134 operands[2], operands[3],
3135 operands[4], p));
3136 DONE;
3137})
3138
3139(define_expand "aarch64_sqdmlsl2_lane<mode>"
3140 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3141 (match_operand:<VWIDE> 1 "register_operand" "w")
3142 (match_operand:VQ_HSI 2 "register_operand" "w")
278821f2 3143 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
43e9d192
IB		 3144 (match_operand:SI 4 "immediate_operand" "i")]
3145 "TARGET_SIMD"
3146{
3147 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
43e9d192
IB		 3148 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3149 operands[2], operands[3],
3150 operands[4], p));
3151 DONE;
3152})
3153
3154(define_expand "aarch64_sqdmlsl2_laneq<mode>"
3155 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3156 (match_operand:<VWIDE> 1 "register_operand" "w")
3157 (match_operand:VQ_HSI 2 "register_operand" "w")
278821f2 3158 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
43e9d192
IB		 3159 (match_operand:SI 4 "immediate_operand" "i")]
3160 "TARGET_SIMD"
3161{
3162 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
278821f2 3163 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
43e9d192
IB		 3164 operands[2], operands[3],
3165 operands[4], p));
3166 DONE;
3167})
3168
3169(define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3170 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3171 (SBINQOPS:<VWIDE>
3172 (match_operand:<VWIDE> 1 "register_operand" "0")
3173 (ss_ashift:<VWIDE>
3174 (mult:<VWIDE>
3175 (sign_extend:<VWIDE>
3176 (vec_select:<VHALF>
3177 (match_operand:VQ_HSI 2 "register_operand" "w")
3178 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3179 (sign_extend:<VWIDE>
3180 (vec_duplicate:<VHALF>
1c83b673 3181 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
43e9d192
IB		 3182 (const_int 1))))]
3183 "TARGET_SIMD"
3184 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
78ec3036 3185 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
43e9d192
IB		 3186)
3187
3188(define_expand "aarch64_sqdmlal2_n<mode>"
3189 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3190 (match_operand:<VWIDE> 1 "register_operand" "w")
3191 (match_operand:VQ_HSI 2 "register_operand" "w")
3192 (match_operand:<VEL> 3 "register_operand" "w")]
3193 "TARGET_SIMD"
3194{
3195 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3196 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3197 operands[2], operands[3],
3198 p));
3199 DONE;
3200})
3201
3202(define_expand "aarch64_sqdmlsl2_n<mode>"
3203 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3204 (match_operand:<VWIDE> 1 "register_operand" "w")
3205 (match_operand:VQ_HSI 2 "register_operand" "w")
3206 (match_operand:<VEL> 3 "register_operand" "w")]
3207 "TARGET_SIMD"
3208{
3209 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3210 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3211 operands[2], operands[3],
3212 p));
3213 DONE;
3214})
3215
3216;; vqdmull
3217
3218(define_insn "aarch64_sqdmull<mode>"
3219 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3220 (ss_ashift:<VWIDE>
3221 (mult:<VWIDE>
3222 (sign_extend:<VWIDE>
3223 (match_operand:VSD_HSI 1 "register_operand" "w"))
3224 (sign_extend:<VWIDE>
3225 (match_operand:VSD_HSI 2 "register_operand" "w")))
3226 (const_int 1)))]
3227 "TARGET_SIMD"
3228 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 3229 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
43e9d192
IB		 3230)
3231
3232;; vqdmull_lane
3233
2a49c16d 3234(define_insn "aarch64_sqdmull_lane<mode>"
43e9d192
IB		 3235 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3236 (ss_ashift:<VWIDE>
3237 (mult:<VWIDE>
3238 (sign_extend:<VWIDE>
3239 (match_operand:VD_HSI 1 "register_operand" "w"))
3240 (sign_extend:<VWIDE>
3241 (vec_duplicate:VD_HSI
3242 (vec_select:<VEL>
278821f2
KT		 3243 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3244 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3245 ))
3246 (const_int 1)))]
3247 "TARGET_SIMD"
3248 {
3249 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3250 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3251 }
3252 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3253)
3254
2a49c16d 3255(define_insn "aarch64_sqdmull_laneq<mode>"
278821f2
KT		 3256 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3257 (ss_ashift:<VWIDE>
3258 (mult:<VWIDE>
3259 (sign_extend:<VWIDE>
3260 (match_operand:VD_HSI 1 "register_operand" "w"))
3261 (sign_extend:<VWIDE>
3262 (vec_duplicate:VD_HSI
3263 (vec_select:<VEL>
3264 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
43e9d192
IB		 3265 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3266 ))
3267 (const_int 1)))]
3268 "TARGET_SIMD"
1dd055a2
JG		 3269 {
3270 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3271 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3272 }
78ec3036 3273 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
43e9d192
IB		 3274)
3275
2a49c16d 3276(define_insn "aarch64_sqdmull_lane<mode>"
43e9d192
IB		 3277 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3278 (ss_ashift:<VWIDE>
3279 (mult:<VWIDE>
3280 (sign_extend:<VWIDE>
3281 (match_operand:SD_HSI 1 "register_operand" "w"))
3282 (sign_extend:<VWIDE>
3283 (vec_select:<VEL>
278821f2
KT		 3284 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3285 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3286 ))
3287 (const_int 1)))]
3288 "TARGET_SIMD"
3289 {
3290 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3291 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3292 }
3293 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3294)
3295
2a49c16d 3296(define_insn "aarch64_sqdmull_laneq<mode>"
278821f2
KT		 3297 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3298 (ss_ashift:<VWIDE>
3299 (mult:<VWIDE>
3300 (sign_extend:<VWIDE>
3301 (match_operand:SD_HSI 1 "register_operand" "w"))
3302 (sign_extend:<VWIDE>
3303 (vec_select:<VEL>
3304 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
43e9d192
IB		 3305 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3306 ))
3307 (const_int 1)))]
3308 "TARGET_SIMD"
1dd055a2
JG		 3309 {
3310 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3311 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3312 }
78ec3036 3313 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
43e9d192
IB		 3314)
3315
43e9d192
IB		 3316;; vqdmull_n
3317
3318(define_insn "aarch64_sqdmull_n<mode>"
3319 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3320 (ss_ashift:<VWIDE>
3321 (mult:<VWIDE>
3322 (sign_extend:<VWIDE>
3323 (match_operand:VD_HSI 1 "register_operand" "w"))
3324 (sign_extend:<VWIDE>
3325 (vec_duplicate:VD_HSI
1c83b673 3326 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
43e9d192
IB		 3327 )
3328 (const_int 1)))]
3329 "TARGET_SIMD"
3330 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
78ec3036 3331 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
43e9d192
IB		 3332)
3333
3334;; vqdmull2
3335
3336
3337
3338(define_insn "aarch64_sqdmull2<mode>_internal"
3339 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3340 (ss_ashift:<VWIDE>
3341 (mult:<VWIDE>
3342 (sign_extend:<VWIDE>
3343 (vec_select:<VHALF>
3344 (match_operand:VQ_HSI 1 "register_operand" "w")
3345 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3346 (sign_extend:<VWIDE>
3347 (vec_select:<VHALF>
3348 (match_operand:VQ_HSI 2 "register_operand" "w")
3349 (match_dup 3)))
3350 )
3351 (const_int 1)))]
3352 "TARGET_SIMD"
3353 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 3354 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
43e9d192
IB		 3355)
3356
3357(define_expand "aarch64_sqdmull2<mode>"
3358 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3359 (match_operand:VQ_HSI 1 "register_operand" "w")
278821f2 3360 (match_operand:VQ_HSI 2 "register_operand" "w")]
43e9d192
IB		 3361 "TARGET_SIMD"
3362{
3363 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3364 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3365 operands[2], p));
3366 DONE;
3367})
3368
3369;; vqdmull2_lane
3370
3371(define_insn "aarch64_sqdmull2_lane<mode>_internal"
3372 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3373 (ss_ashift:<VWIDE>
3374 (mult:<VWIDE>
3375 (sign_extend:<VWIDE>
3376 (vec_select:<VHALF>
3377 (match_operand:VQ_HSI 1 "register_operand" "w")
3378 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3379 (sign_extend:<VWIDE>
3380 (vec_duplicate:<VHALF>
3381 (vec_select:<VEL>
278821f2
KT		 3382 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3383 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3384 ))
3385 (const_int 1)))]
3386 "TARGET_SIMD"
3387 {
3388 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3389 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3390 }
3391 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3392)
3393
3394(define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3395 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3396 (ss_ashift:<VWIDE>
3397 (mult:<VWIDE>
3398 (sign_extend:<VWIDE>
3399 (vec_select:<VHALF>
3400 (match_operand:VQ_HSI 1 "register_operand" "w")
3401 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3402 (sign_extend:<VWIDE>
3403 (vec_duplicate:<VHALF>
3404 (vec_select:<VEL>
3405 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
43e9d192
IB		 3406 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3407 ))
3408 (const_int 1)))]
3409 "TARGET_SIMD"
1dd055a2
JG		 3410 {
3411 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3412 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3413 }
78ec3036 3414 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
43e9d192
IB		 3415)
3416
3417(define_expand "aarch64_sqdmull2_lane<mode>"
3418 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3419 (match_operand:VQ_HSI 1 "register_operand" "w")
278821f2 3420 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
43e9d192
IB		 3421 (match_operand:SI 3 "immediate_operand" "i")]
3422 "TARGET_SIMD"
3423{
3424 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
43e9d192
IB		 3425 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3426 operands[2], operands[3],
3427 p));
3428 DONE;
3429})
3430
3431(define_expand "aarch64_sqdmull2_laneq<mode>"
3432 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3433 (match_operand:VQ_HSI 1 "register_operand" "w")
278821f2 3434 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
43e9d192
IB		 3435 (match_operand:SI 3 "immediate_operand" "i")]
3436 "TARGET_SIMD"
3437{
3438 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
278821f2 3439 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
43e9d192
IB		 3440 operands[2], operands[3],
3441 p));
3442 DONE;
3443})
3444
3445;; vqdmull2_n
3446
3447(define_insn "aarch64_sqdmull2_n<mode>_internal"
3448 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3449 (ss_ashift:<VWIDE>
3450 (mult:<VWIDE>
3451 (sign_extend:<VWIDE>
3452 (vec_select:<VHALF>
3453 (match_operand:VQ_HSI 1 "register_operand" "w")
3454 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3455 (sign_extend:<VWIDE>
3456 (vec_duplicate:<VHALF>
1c83b673 3457 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
43e9d192
IB		 3458 )
3459 (const_int 1)))]
3460 "TARGET_SIMD"
3461 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
78ec3036 3462 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
43e9d192
IB		 3463)
3464
3465(define_expand "aarch64_sqdmull2_n<mode>"
3466 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3467 (match_operand:VQ_HSI 1 "register_operand" "w")
3468 (match_operand:<VEL> 2 "register_operand" "w")]
3469 "TARGET_SIMD"
3470{
3471 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3472 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3473 operands[2], p));
3474 DONE;
3475})
3476
3477;; vshl
3478
3479(define_insn "aarch64_<sur>shl<mode>"
3480 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3481 (unspec:VSDQ_I_DI
3482 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3483 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3484 VSHL))]
3485 "TARGET_SIMD"
3486 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
78ec3036 3487 [(set_attr "type" "neon_shift_reg<q>")]
43e9d192
IB		 3488)
3489
3490
3491;; vqshl
3492
3493(define_insn "aarch64_<sur>q<r>shl<mode>"
3494 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3495 (unspec:VSDQ_I
3496 [(match_operand:VSDQ_I 1 "register_operand" "w")
3497 (match_operand:VSDQ_I 2 "register_operand" "w")]
3498 VQSHL))]
3499 "TARGET_SIMD"
3500 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
78ec3036 3501 [(set_attr "type" "neon_sat_shift_reg<q>")]
43e9d192
IB		 3502)
3503
43e9d192
IB		 3504;; vshll_n
3505
3506(define_insn "aarch64_<sur>shll_n<mode>"
3507 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
a844a695 3508 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
cb23a30c
JG		 3509 (match_operand:SI 2
3510 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
43e9d192
IB		 3511 VSHLL))]
3512 "TARGET_SIMD"
3513 "*
3514 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
43e9d192
IB		 3515 if (INTVAL (operands[2]) == bit_width)
3516 {
3517 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3518 }
3519 else {
3520 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3521 }"
78ec3036 3522 [(set_attr "type" "neon_shift_imm_long")]
43e9d192
IB		 3523)
3524
3525;; vshll_high_n
3526
3527(define_insn "aarch64_<sur>shll2_n<mode>"
3528 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3529 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3530 (match_operand:SI 2 "immediate_operand" "i")]
3531 VSHLL))]
3532 "TARGET_SIMD"
3533 "*
3534 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
43e9d192
IB		 3535 if (INTVAL (operands[2]) == bit_width)
3536 {
3537 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3538 }
3539 else {
3540 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3541 }"
78ec3036 3542 [(set_attr "type" "neon_shift_imm_long")]
43e9d192
IB		 3543)
3544
43e9d192
IB		 3545;; vrshr_n
3546
3547(define_insn "aarch64_<sur>shr_n<mode>"
3548 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3549 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
cb23a30c
JG		 3550 (match_operand:SI 2
3551 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
43e9d192
IB		 3552 VRSHR_N))]
3553 "TARGET_SIMD"
cb23a30c 3554 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
78ec3036 3555 [(set_attr "type" "neon_sat_shift_imm<q>")]
43e9d192
IB		 3556)
3557
3558;; v(r)sra_n
3559
3560(define_insn "aarch64_<sur>sra_n<mode>"
3561 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3562 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3563 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
cb23a30c
JG		 3564 (match_operand:SI 3
3565 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
43e9d192
IB		 3566 VSRA))]
3567 "TARGET_SIMD"
cb23a30c 3568 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
78ec3036 3569 [(set_attr "type" "neon_shift_acc<q>")]
43e9d192
IB		 3570)
3571
3572;; vs<lr>i_n
3573
3574(define_insn "aarch64_<sur>s<lr>i_n<mode>"
3575 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3576 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3577 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
cb23a30c
JG		 3578 (match_operand:SI 3
3579 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
43e9d192
IB		 3580 VSLRI))]
3581 "TARGET_SIMD"
cb23a30c 3582 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
78ec3036 3583 [(set_attr "type" "neon_shift_imm<q>")]
43e9d192
IB		 3584)
3585
3586;; vqshl(u)
3587
3588(define_insn "aarch64_<sur>qshl<u>_n<mode>"
3589 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3590 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
cb23a30c
JG		 3591 (match_operand:SI 2
3592 "aarch64_simd_shift_imm_<ve_mode>" "i")]
43e9d192
IB		 3593 VQSHL_N))]
3594 "TARGET_SIMD"
cb23a30c 3595 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
78ec3036 3596 [(set_attr "type" "neon_sat_shift_imm<q>")]
43e9d192
IB		 3597)
3598
3599
3600;; vq(r)shr(u)n_n
3601
3602(define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3603 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3604 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
cb23a30c
JG		 3605 (match_operand:SI 2
3606 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
43e9d192
IB		 3607 VQSHRN_N))]
3608 "TARGET_SIMD"
cb23a30c 3609 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
78ec3036 3610 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
43e9d192
IB		 3611)
3612
3613
889b9412
JG		 3614;; cm(eq|ge|gt|lt|le)
3615;; Note, we have constraints for Dz and Z as different expanders
3616;; have different ideas of what should be passed to this pattern.
43e9d192 3617
889b9412 3618(define_insn "aarch64_cm<optab><mode>"
43e9d192 3619 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
889b9412
JG		 3620 (neg:<V_cmp_result>
3621 (COMPARISONS:<V_cmp_result>
a844a695
AL		 3622 (match_operand:VDQ_I 1 "register_operand" "w,w")
3623 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
889b9412 3624 )))]
43e9d192
IB		 3625 "TARGET_SIMD"
3626 "@
889b9412
JG		 3627 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3628 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
78ec3036 3629 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
43e9d192
IB		 3630)
3631
889b9412
JG		 3632(define_insn_and_split "aarch64_cm<optab>di"
3633 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3634 (neg:DI
3635 (COMPARISONS:DI
3636 (match_operand:DI 1 "register_operand" "w,w,r")
3637 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
75c7257f
JG		 3638 )))
3639 (clobber (reg:CC CC_REGNUM))]
889b9412 3640 "TARGET_SIMD"
110e1ccc
JG		 3641 "#"
3642 "reload_completed"
3643 [(set (match_operand:DI 0 "register_operand")
3644 (neg:DI
3645 (COMPARISONS:DI
3646 (match_operand:DI 1 "register_operand")
3647 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3648 )))]
889b9412 3649 {
110e1ccc
JG		 3650 /* If we are in the general purpose register file,
3651 we split to a sequence of comparison and store. */
3652 if (GP_REGNUM_P (REGNO (operands[0]))
3653 && GP_REGNUM_P (REGNO (operands[1])))
3654 {
ef4bddc2 3655 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
110e1ccc
JG		 3656 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3657 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3658 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3659 DONE;
3660 }
3661 /* Otherwise, we expand to a similar pattern which does not
3662 clobber CC_REGNUM. */
889b9412 3663 }
78ec3036 3664 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
889b9412
JG		 3665)
3666
110e1ccc
JG		 3667(define_insn "*aarch64_cm<optab>di"
3668 [(set (match_operand:DI 0 "register_operand" "=w,w")
3669 (neg:DI
3670 (COMPARISONS:DI
3671 (match_operand:DI 1 "register_operand" "w,w")
3672 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3673 )))]
3674 "TARGET_SIMD && reload_completed"
3675 "@
3676 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3677 cm<optab>\t%d0, %d1, #0"
3678 [(set_attr "type" "neon_compare, neon_compare_zero")]
3679)
3680
889b9412 3681;; cm(hs|hi)
43e9d192 3682
889b9412 3683(define_insn "aarch64_cm<optab><mode>"
43e9d192 3684 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
889b9412
JG		 3685 (neg:<V_cmp_result>
3686 (UCOMPARISONS:<V_cmp_result>
a844a695
AL		 3687 (match_operand:VDQ_I 1 "register_operand" "w")
3688 (match_operand:VDQ_I 2 "register_operand" "w")
889b9412 3689 )))]
43e9d192 3690 "TARGET_SIMD"
889b9412 3691 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
78ec3036 3692 [(set_attr "type" "neon_compare<q>")]
43e9d192
IB		 3693)
3694
889b9412
JG		 3695(define_insn_and_split "aarch64_cm<optab>di"
3696 [(set (match_operand:DI 0 "register_operand" "=w,r")
3697 (neg:DI
3698 (UCOMPARISONS:DI
3699 (match_operand:DI 1 "register_operand" "w,r")
3700 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
75c7257f
JG		 3701 )))
3702 (clobber (reg:CC CC_REGNUM))]
889b9412 3703 "TARGET_SIMD"
110e1ccc
JG		 3704 "#"
3705 "reload_completed"
3706 [(set (match_operand:DI 0 "register_operand")
3707 (neg:DI
3708 (UCOMPARISONS:DI
3709 (match_operand:DI 1 "register_operand")
3710 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3711 )))]
889b9412 3712 {
110e1ccc
JG		 3713 /* If we are in the general purpose register file,
3714 we split to a sequence of comparison and store. */
3715 if (GP_REGNUM_P (REGNO (operands[0]))
3716 && GP_REGNUM_P (REGNO (operands[1])))
3717 {
ef4bddc2 3718 machine_mode mode = CCmode;
110e1ccc
JG		 3719 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3720 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3721 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3722 DONE;
3723 }
3724 /* Otherwise, we expand to a similar pattern which does not
3725 clobber CC_REGNUM. */
889b9412 3726 }
110e1ccc
JG		 3727 [(set_attr "type" "neon_compare,multiple")]
3728)
3729
3730(define_insn "*aarch64_cm<optab>di"
3731 [(set (match_operand:DI 0 "register_operand" "=w")
3732 (neg:DI
3733 (UCOMPARISONS:DI
3734 (match_operand:DI 1 "register_operand" "w")
3735 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
3736 )))]
3737 "TARGET_SIMD && reload_completed"
3738 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
3739 [(set_attr "type" "neon_compare")]
889b9412 3740)
385eb93d 3741
889b9412
JG		 3742;; cmtst
3743
ddeabd3e
AL		 3744;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
3745;; we don't have any insns using ne, and aarch64_vcond_internal outputs
3746;; not (neg (eq (and x y) 0))
3747;; which is rewritten by simplify_rtx as
3748;; plus (eq (and x y) 0) -1.
3749
889b9412
JG		 3750(define_insn "aarch64_cmtst<mode>"
3751 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
ddeabd3e
AL		 3752 (plus:<V_cmp_result>
3753 (eq:<V_cmp_result>
a844a695
AL		 3754 (and:VDQ_I
3755 (match_operand:VDQ_I 1 "register_operand" "w")
3756 (match_operand:VDQ_I 2 "register_operand" "w"))
3757 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
ddeabd3e
AL		 3758 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
3759 ]
889b9412
JG		 3760 "TARGET_SIMD"
3761 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 3762 [(set_attr "type" "neon_tst<q>")]
889b9412
JG		 3763)
3764
3765(define_insn_and_split "aarch64_cmtstdi"
3766 [(set (match_operand:DI 0 "register_operand" "=w,r")
3767 (neg:DI
3768 (ne:DI
3769 (and:DI
3770 (match_operand:DI 1 "register_operand" "w,r")
3771 (match_operand:DI 2 "register_operand" "w,r"))
75c7257f
JG		 3772 (const_int 0))))
3773 (clobber (reg:CC CC_REGNUM))]
889b9412 3774 "TARGET_SIMD"
110e1ccc
JG		 3775 "#"
3776 "reload_completed"
3777 [(set (match_operand:DI 0 "register_operand")
3778 (neg:DI
3779 (ne:DI
3780 (and:DI
3781 (match_operand:DI 1 "register_operand")
3782 (match_operand:DI 2 "register_operand"))
3783 (const_int 0))))]
889b9412 3784 {
110e1ccc
JG		 3785 /* If we are in the general purpose register file,
3786 we split to a sequence of comparison and store. */
3787 if (GP_REGNUM_P (REGNO (operands[0]))
3788 && GP_REGNUM_P (REGNO (operands[1])))
3789 {
3790 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
ef4bddc2 3791 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
110e1ccc
JG		 3792 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3793 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3794 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3795 DONE;
3796 }
3797 /* Otherwise, we expand to a similar pattern which does not
3798 clobber CC_REGNUM. */
889b9412 3799 }
110e1ccc
JG		 3800 [(set_attr "type" "neon_tst,multiple")]
3801)
3802
3803(define_insn "*aarch64_cmtstdi"
3804 [(set (match_operand:DI 0 "register_operand" "=w")
3805 (neg:DI
3806 (ne:DI
3807 (and:DI
3808 (match_operand:DI 1 "register_operand" "w")
3809 (match_operand:DI 2 "register_operand" "w"))
3810 (const_int 0))))]
3811 "TARGET_SIMD"
3812 "cmtst\t%d0, %d1, %d2"
78ec3036 3813 [(set_attr "type" "neon_tst")]
889b9412
JG		 3814)
3815
3816;; fcm(eq|ge|gt|le|lt)
3817
3818(define_insn "aarch64_cm<optab><mode>"
385eb93d 3819 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
889b9412
JG		 3820 (neg:<V_cmp_result>
3821 (COMPARISONS:<V_cmp_result>
3822 (match_operand:VALLF 1 "register_operand" "w,w")
3823 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3824 )))]
385eb93d
JG		 3825 "TARGET_SIMD"
3826 "@
889b9412
JG		 3827 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3828 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
78ec3036 3829 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
385eb93d
JG		 3830)
3831
75dd5ace
JG		 3832;; fac(ge|gt)
3833;; Note we can also handle what would be fac(le|lt) by
3834;; generating fac(ge|gt).
3835
3836(define_insn "*aarch64_fac<optab><mode>"
3837 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3838 (neg:<V_cmp_result>
3839 (FAC_COMPARISONS:<V_cmp_result>
3840 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3841 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3842 )))]
3843 "TARGET_SIMD"
3844 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
78ec3036 3845 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
75dd5ace
JG		 3846)
3847
43e9d192
IB		 3848;; addp
3849
3850(define_insn "aarch64_addp<mode>"
3851 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3852 (unspec:VD_BHSI
3853 [(match_operand:VD_BHSI 1 "register_operand" "w")
3854 (match_operand:VD_BHSI 2 "register_operand" "w")]
3855 UNSPEC_ADDP))]
3856 "TARGET_SIMD"
3857 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 3858 [(set_attr "type" "neon_reduc_add<q>")]
43e9d192
IB		 3859)
3860
3861(define_insn "aarch64_addpdi"
3862 [(set (match_operand:DI 0 "register_operand" "=w")
3863 (unspec:DI
3864 [(match_operand:V2DI 1 "register_operand" "w")]
3865 UNSPEC_ADDP))]
3866 "TARGET_SIMD"
3867 "addp\t%d0, %1.2d"
78ec3036 3868 [(set_attr "type" "neon_reduc_add")]
43e9d192
IB		 3869)
3870
43e9d192
IB		 3871;; sqrt
3872
3873(define_insn "sqrt<mode>2"
3874 [(set (match_operand:VDQF 0 "register_operand" "=w")
3875 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3876 "TARGET_SIMD"
3877 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 3878 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
43e9d192
IB		 3879)
3880
43e9d192
IB		 3881;; Patterns for vector struct loads and stores.
3882
668046d1 3883(define_insn "aarch64_simd_ld2<mode>"
43e9d192
IB		 3884 [(set (match_operand:OI 0 "register_operand" "=w")
3885 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3886 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3887 UNSPEC_LD2))]
3888 "TARGET_SIMD"
3889 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
78ec3036
JG		 3890 [(set_attr "type" "neon_load2_2reg<q>")]
3891)
43e9d192 3892
77efea31
FY		 3893(define_insn "aarch64_simd_ld2r<mode>"
3894 [(set (match_operand:OI 0 "register_operand" "=w")
3895 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3896 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3897 UNSPEC_LD2_DUP))]
3898 "TARGET_SIMD"
3899 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3900 [(set_attr "type" "neon_load2_all_lanes<q>")]
3901)
3902
3ec1be97
CB		 3903(define_insn "aarch64_vec_load_lanesoi_lane<mode>"
3904 [(set (match_operand:OI 0 "register_operand" "=w")
3905 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3906 (match_operand:OI 2 "register_operand" "0")
3907 (match_operand:SI 3 "immediate_operand" "i")
3908 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3909 UNSPEC_LD2_LANE))]
3910 "TARGET_SIMD"
3911 "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1"
3912 [(set_attr "type" "neon_load2_one_lane")]
3913)
3914
668046d1
DS		 3915(define_expand "vec_load_lanesoi<mode>"
3916 [(set (match_operand:OI 0 "register_operand" "=w")
3917 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3918 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3919 UNSPEC_LD2))]
3920 "TARGET_SIMD"
3921{
3922 if (BYTES_BIG_ENDIAN)
3923 {
3924 rtx tmp = gen_reg_rtx (OImode);
3925 rtx mask = aarch64_reverse_mask (<MODE>mode);
3926 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
3927 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
3928 }
3929 else
3930 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
3931 DONE;
3932})
3933
3934(define_insn "aarch64_simd_st2<mode>"
43e9d192
IB		 3935 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3936 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3937 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3938 UNSPEC_ST2))]
3939 "TARGET_SIMD"
3940 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
78ec3036
JG		 3941 [(set_attr "type" "neon_store2_2reg<q>")]
3942)
43e9d192 3943
ba081b77
JG		 3944(define_insn "vec_store_lanesoi_lane<mode>"
3945 [(set (match_operand:<V_TWO_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
3946 (unspec:<V_TWO_ELEM> [(match_operand:OI 1 "register_operand" "w")
3947 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
3948 (match_operand:SI 2 "immediate_operand" "i")]
3949 UNSPEC_ST2_LANE))]
3950 "TARGET_SIMD"
3951 "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0"
3952 [(set_attr "type" "neon_store3_one_lane<q>")]
3953)
3954
668046d1
DS		 3955(define_expand "vec_store_lanesoi<mode>"
3956 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3957 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3958 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3959 UNSPEC_ST2))]
3960 "TARGET_SIMD"
3961{
3962 if (BYTES_BIG_ENDIAN)
3963 {
3964 rtx tmp = gen_reg_rtx (OImode);
3965 rtx mask = aarch64_reverse_mask (<MODE>mode);
3966 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
3967 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
3968 }
3969 else
3970 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
3971 DONE;
3972})
3973
3974(define_insn "aarch64_simd_ld3<mode>"
43e9d192
IB		 3975 [(set (match_operand:CI 0 "register_operand" "=w")
3976 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3977 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3978 UNSPEC_LD3))]
3979 "TARGET_SIMD"
3980 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
78ec3036
JG		 3981 [(set_attr "type" "neon_load3_3reg<q>")]
3982)
43e9d192 3983
77efea31
FY		 3984(define_insn "aarch64_simd_ld3r<mode>"
3985 [(set (match_operand:CI 0 "register_operand" "=w")
3986 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3987 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3988 UNSPEC_LD3_DUP))]
3989 "TARGET_SIMD"
3990 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3991 [(set_attr "type" "neon_load3_all_lanes<q>")]
3992)
3993
3ec1be97
CB		 3994(define_insn "aarch64_vec_load_lanesci_lane<mode>"
3995 [(set (match_operand:CI 0 "register_operand" "=w")
3996 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3997 (match_operand:CI 2 "register_operand" "0")
3998 (match_operand:SI 3 "immediate_operand" "i")
3999 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4000 UNSPEC_LD3_LANE))]
4001 "TARGET_SIMD"
4002 "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1"
4003 [(set_attr "type" "neon_load3_one_lane")]
4004)
4005
668046d1
DS		 4006(define_expand "vec_load_lanesci<mode>"
4007 [(set (match_operand:CI 0 "register_operand" "=w")
4008 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4009 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4010 UNSPEC_LD3))]
4011 "TARGET_SIMD"
4012{
4013 if (BYTES_BIG_ENDIAN)
4014 {
4015 rtx tmp = gen_reg_rtx (CImode);
4016 rtx mask = aarch64_reverse_mask (<MODE>mode);
4017 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4018 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4019 }
4020 else
4021 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4022 DONE;
4023})
4024
4025(define_insn "aarch64_simd_st3<mode>"
43e9d192
IB		 4026 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4027 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4028 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4029 UNSPEC_ST3))]
4030 "TARGET_SIMD"
4031 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
78ec3036
JG		 4032 [(set_attr "type" "neon_store3_3reg<q>")]
4033)
43e9d192 4034
ba081b77
JG		 4035(define_insn "vec_store_lanesci_lane<mode>"
4036 [(set (match_operand:<V_THREE_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4037 (unspec:<V_THREE_ELEM> [(match_operand:CI 1 "register_operand" "w")
4038 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4039 (match_operand:SI 2 "immediate_operand" "i")]
4040 UNSPEC_ST3_LANE))]
4041 "TARGET_SIMD"
4042 "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0"
4043 [(set_attr "type" "neon_store3_one_lane<q>")]
4044)
4045
668046d1
DS		 4046(define_expand "vec_store_lanesci<mode>"
4047 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4048 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4049 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4050 UNSPEC_ST3))]
4051 "TARGET_SIMD"
4052{
4053 if (BYTES_BIG_ENDIAN)
4054 {
4055 rtx tmp = gen_reg_rtx (CImode);
4056 rtx mask = aarch64_reverse_mask (<MODE>mode);
4057 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4058 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4059 }
4060 else
4061 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4062 DONE;
4063})
4064
4065(define_insn "aarch64_simd_ld4<mode>"
43e9d192
IB		 4066 [(set (match_operand:XI 0 "register_operand" "=w")
4067 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4068 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4069 UNSPEC_LD4))]
4070 "TARGET_SIMD"
4071 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
78ec3036
JG		 4072 [(set_attr "type" "neon_load4_4reg<q>")]
4073)
43e9d192 4074
77efea31
FY		 4075(define_insn "aarch64_simd_ld4r<mode>"
4076 [(set (match_operand:XI 0 "register_operand" "=w")
4077 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4078 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4079 UNSPEC_LD4_DUP))]
4080 "TARGET_SIMD"
4081 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4082 [(set_attr "type" "neon_load4_all_lanes<q>")]
4083)
4084
3ec1be97
CB		 4085(define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4086 [(set (match_operand:XI 0 "register_operand" "=w")
4087 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4088 (match_operand:XI 2 "register_operand" "0")
4089 (match_operand:SI 3 "immediate_operand" "i")
4090 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4091 UNSPEC_LD4_LANE))]
4092 "TARGET_SIMD"
4093 "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1"
4094 [(set_attr "type" "neon_load4_one_lane")]
4095)
4096
668046d1
DS		 4097(define_expand "vec_load_lanesxi<mode>"
4098 [(set (match_operand:XI 0 "register_operand" "=w")
4099 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4100 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4101 UNSPEC_LD4))]
4102 "TARGET_SIMD"
4103{
4104 if (BYTES_BIG_ENDIAN)
4105 {
4106 rtx tmp = gen_reg_rtx (XImode);
4107 rtx mask = aarch64_reverse_mask (<MODE>mode);
4108 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4109 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4110 }
4111 else
4112 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4113 DONE;
4114})
4115
4116(define_insn "aarch64_simd_st4<mode>"
43e9d192
IB		 4117 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4118 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4119 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4120 UNSPEC_ST4))]
4121 "TARGET_SIMD"
4122 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
78ec3036
JG		 4123 [(set_attr "type" "neon_store4_4reg<q>")]
4124)
43e9d192 4125
ba081b77
JG		 4126(define_insn "vec_store_lanesxi_lane<mode>"
4127 [(set (match_operand:<V_FOUR_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4128 (unspec:<V_FOUR_ELEM> [(match_operand:XI 1 "register_operand" "w")
4129 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4130 (match_operand:SI 2 "immediate_operand" "i")]
4131 UNSPEC_ST4_LANE))]
4132 "TARGET_SIMD"
4133 "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0"
4134 [(set_attr "type" "neon_store4_one_lane<q>")]
4135)
4136
668046d1
DS		 4137(define_expand "vec_store_lanesxi<mode>"
4138 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4139 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4140 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4141 UNSPEC_ST4))]
4142 "TARGET_SIMD"
4143{
4144 if (BYTES_BIG_ENDIAN)
4145 {
4146 rtx tmp = gen_reg_rtx (XImode);
4147 rtx mask = aarch64_reverse_mask (<MODE>mode);
4148 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4149 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4150 }
4151 else
4152 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4153 DONE;
4154})
4155
4156(define_insn_and_split "aarch64_rev_reglist<mode>"
4157[(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4158 (unspec:VSTRUCT
4159 [(match_operand:VSTRUCT 1 "register_operand" "w")
4160 (match_operand:V16QI 2 "register_operand" "w")]
4161 UNSPEC_REV_REGLIST))]
4162 "TARGET_SIMD"
4163 "#"
4164 "&& reload_completed"
4165 [(const_int 0)]
4166{
4167 int i;
4168 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4169 for (i = 0; i < nregs; i++)
4170 {
4171 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4172 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4173 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4174 }
4175 DONE;
4176}
4177 [(set_attr "type" "neon_tbl1_q")
4178 (set_attr "length" "<insn_count>")]
4179)
4180
43e9d192
IB		 4181;; Reload patterns for AdvSIMD register list operands.
4182
4183(define_expand "mov<mode>"
2d8c6dc1
AH		 4184 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4185 (match_operand:VSTRUCT 1 "general_operand" ""))]
43e9d192
IB		 4186 "TARGET_SIMD"
4187{
4188 if (can_create_pseudo_p ())
4189 {
4190 if (GET_CODE (operands[0]) != REG)
4191 operands[1] = force_reg (<MODE>mode, operands[1]);
4192 }
4193})
4194
4195(define_insn "*aarch64_mov<mode>"
4196 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
2d8c6dc1
AH		 4197 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4198 "TARGET_SIMD && !BYTES_BIG_ENDIAN
43e9d192
IB		 4199 && (register_operand (operands[0], <MODE>mode)
4200 || register_operand (operands[1], <MODE>mode))"
2d8c6dc1
AH		 4201 "@
4202 #
4203 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4204 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4205 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4206 neon_load<nregs>_<nregs>reg_q")
78ec3036
JG		 4207 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4208)
43e9d192 4209
89b4515c
AV		 4210(define_insn "aarch64_be_ld1<mode>"
4211 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4212 (unspec:VALLDI [(match_operand:VALLDI 1 "aarch64_simd_struct_operand" "Utv")]
4213 UNSPEC_LD1))]
4214 "TARGET_SIMD"
4215 "ld1\\t{%0<Vmtype>}, %1"
4216 [(set_attr "type" "neon_load1_1reg<q>")]
4217)
4218
4219(define_insn "aarch64_be_st1<mode>"
4220 [(set (match_operand:VALLDI 0 "aarch64_simd_struct_operand" "=Utv")
4221 (unspec:VALLDI [(match_operand:VALLDI 1 "register_operand" "w")]
4222 UNSPEC_ST1))]
4223 "TARGET_SIMD"
4224 "st1\\t{%1<Vmtype>}, %0"
4225 [(set_attr "type" "neon_store1_1reg<q>")]
4226)
4227
2d8c6dc1
AH		 4228(define_insn "*aarch64_be_movoi"
4229 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4230 (match_operand:OI 1 "general_operand" " w,w,m"))]
4231 "TARGET_SIMD && BYTES_BIG_ENDIAN
4232 && (register_operand (operands[0], OImode)
4233 || register_operand (operands[1], OImode))"
4234 "@
4235 #
4236 stp\\t%q1, %R1, %0
4237 ldp\\t%q0, %R0, %1"
4238 [(set_attr "type" "multiple,neon_store2_2reg_q,neon_load2_2reg_q")
4239 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4240)
4241
4242(define_insn "*aarch64_be_movci"
4243 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4244 (match_operand:CI 1 "general_operand" " w,w,o"))]
4245 "TARGET_SIMD && BYTES_BIG_ENDIAN
4246 && (register_operand (operands[0], CImode)
4247 || register_operand (operands[1], CImode))"
4248 "#"
4249 [(set_attr "type" "multiple")
4250 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4251)
4252
4253(define_insn "*aarch64_be_movxi"
4254 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4255 (match_operand:XI 1 "general_operand" " w,w,o"))]
4256 "TARGET_SIMD && BYTES_BIG_ENDIAN
4257 && (register_operand (operands[0], XImode)
4258 || register_operand (operands[1], XImode))"
4259 "#"
4260 [(set_attr "type" "multiple")
4261 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4262)
4263
43e9d192 4264(define_split
2d8c6dc1
AH		 4265 [(set (match_operand:OI 0 "register_operand")
4266 (match_operand:OI 1 "register_operand"))]
43e9d192 4267 "TARGET_SIMD && reload_completed"
2d8c6dc1 4268 [(const_int 0)]
43e9d192 4269{
2d8c6dc1
AH		 4270 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4271 DONE;
43e9d192
IB		 4272})
4273
4274(define_split
2d8c6dc1
AH		 4275 [(set (match_operand:CI 0 "nonimmediate_operand")
4276 (match_operand:CI 1 "general_operand"))]
43e9d192 4277 "TARGET_SIMD && reload_completed"
2d8c6dc1 4278 [(const_int 0)]
43e9d192 4279{
2d8c6dc1
AH		 4280 if (register_operand (operands[0], CImode)
4281 && register_operand (operands[1], CImode))
4282 {
4283 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4284 DONE;
4285 }
4286 else if (BYTES_BIG_ENDIAN)
4287 {
4288 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4289 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4290 emit_move_insn (gen_lowpart (V16QImode,
4291 simplify_gen_subreg (TImode, operands[0],
4292 CImode, 32)),
4293 gen_lowpart (V16QImode,
4294 simplify_gen_subreg (TImode, operands[1],
4295 CImode, 32)));
4296 DONE;
4297 }
4298 else
4299 FAIL;
43e9d192
IB		 4300})
4301
4302(define_split
2d8c6dc1
AH		 4303 [(set (match_operand:XI 0 "nonimmediate_operand")
4304 (match_operand:XI 1 "general_operand"))]
43e9d192 4305 "TARGET_SIMD && reload_completed"
2d8c6dc1 4306 [(const_int 0)]
43e9d192 4307{
2d8c6dc1
AH		 4308 if (register_operand (operands[0], XImode)
4309 && register_operand (operands[1], XImode))
4310 {
4311 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4312 DONE;
4313 }
4314 else if (BYTES_BIG_ENDIAN)
4315 {
4316 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4317 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4318 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4319 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4320 DONE;
4321 }
4322 else
4323 FAIL;
43e9d192
IB		 4324})
4325
77efea31
FY		 4326(define_expand "aarch64_ld2r<mode>"
4327 [(match_operand:OI 0 "register_operand" "=w")
4328 (match_operand:DI 1 "register_operand" "w")
4329 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4330 "TARGET_SIMD"
4331{
ef4bddc2 4332 machine_mode mode = <V_TWO_ELEM>mode;
77efea31
FY		 4333 rtx mem = gen_rtx_MEM (mode, operands[1]);
4334
4335 emit_insn (gen_aarch64_simd_ld2r<mode> (operands[0], mem));
4336 DONE;
4337})
4338
4339(define_expand "aarch64_ld3r<mode>"
4340 [(match_operand:CI 0 "register_operand" "=w")
4341 (match_operand:DI 1 "register_operand" "w")
4342 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4343 "TARGET_SIMD"
4344{
ef4bddc2 4345 machine_mode mode = <V_THREE_ELEM>mode;
77efea31
FY		 4346 rtx mem = gen_rtx_MEM (mode, operands[1]);
4347
4348 emit_insn (gen_aarch64_simd_ld3r<mode> (operands[0], mem));
4349 DONE;
4350})
4351
4352(define_expand "aarch64_ld4r<mode>"
4353 [(match_operand:XI 0 "register_operand" "=w")
4354 (match_operand:DI 1 "register_operand" "w")
4355 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4356 "TARGET_SIMD"
4357{
ef4bddc2 4358 machine_mode mode = <V_FOUR_ELEM>mode;
77efea31
FY		 4359 rtx mem = gen_rtx_MEM (mode, operands[1]);
4360
4361 emit_insn (gen_aarch64_simd_ld4r<mode> (operands[0],mem));
4362 DONE;
4363})
4364
43e9d192
IB		 4365(define_insn "aarch64_ld2<mode>_dreg"
4366 [(set (match_operand:OI 0 "register_operand" "=w")
4367 (subreg:OI
4368 (vec_concat:<VRL2>
4369 (vec_concat:<VDBL>
4370 (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4371 UNSPEC_LD2)
4372 (vec_duplicate:VD (const_int 0)))
4373 (vec_concat:<VDBL>
4374 (unspec:VD [(match_dup 1)]
4375 UNSPEC_LD2)
4376 (vec_duplicate:VD (const_int 0)))) 0))]
4377 "TARGET_SIMD"
4378 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
78ec3036
JG		 4379 [(set_attr "type" "neon_load2_2reg<q>")]
4380)
43e9d192
IB		 4381
4382(define_insn "aarch64_ld2<mode>_dreg"
4383 [(set (match_operand:OI 0 "register_operand" "=w")
4384 (subreg:OI
4385 (vec_concat:<VRL2>
4386 (vec_concat:<VDBL>
4387 (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4388 UNSPEC_LD2)
4389 (const_int 0))
4390 (vec_concat:<VDBL>
4391 (unspec:DX [(match_dup 1)]
4392 UNSPEC_LD2)
4393 (const_int 0))) 0))]
4394 "TARGET_SIMD"
4395 "ld1\\t{%S0.1d - %T0.1d}, %1"
78ec3036
JG		 4396 [(set_attr "type" "neon_load1_2reg<q>")]
4397)
43e9d192
IB		 4398
4399(define_insn "aarch64_ld3<mode>_dreg"
4400 [(set (match_operand:CI 0 "register_operand" "=w")
4401 (subreg:CI
4402 (vec_concat:<VRL3>
4403 (vec_concat:<VRL2>
4404 (vec_concat:<VDBL>
4405 (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4406 UNSPEC_LD3)
4407 (vec_duplicate:VD (const_int 0)))
4408 (vec_concat:<VDBL>
4409 (unspec:VD [(match_dup 1)]
4410 UNSPEC_LD3)
4411 (vec_duplicate:VD (const_int 0))))
4412 (vec_concat:<VDBL>
4413 (unspec:VD [(match_dup 1)]
4414 UNSPEC_LD3)
4415 (vec_duplicate:VD (const_int 0)))) 0))]
4416 "TARGET_SIMD"
4417 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
78ec3036
JG		 4418 [(set_attr "type" "neon_load3_3reg<q>")]
4419)
43e9d192
IB		 4420
4421(define_insn "aarch64_ld3<mode>_dreg"
4422 [(set (match_operand:CI 0 "register_operand" "=w")
4423 (subreg:CI
4424 (vec_concat:<VRL3>
4425 (vec_concat:<VRL2>
4426 (vec_concat:<VDBL>
4427 (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4428 UNSPEC_LD3)
4429 (const_int 0))
4430 (vec_concat:<VDBL>
4431 (unspec:DX [(match_dup 1)]
4432 UNSPEC_LD3)
4433 (const_int 0)))
4434 (vec_concat:<VDBL>
4435 (unspec:DX [(match_dup 1)]
4436 UNSPEC_LD3)
4437 (const_int 0))) 0))]
4438 "TARGET_SIMD"
4439 "ld1\\t{%S0.1d - %U0.1d}, %1"
78ec3036
JG		 4440 [(set_attr "type" "neon_load1_3reg<q>")]
4441)
43e9d192
IB		 4442
4443(define_insn "aarch64_ld4<mode>_dreg"
4444 [(set (match_operand:XI 0 "register_operand" "=w")
4445 (subreg:XI
4446 (vec_concat:<VRL4>
4447 (vec_concat:<VRL2>
4448 (vec_concat:<VDBL>
4449 (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4450 UNSPEC_LD4)
4451 (vec_duplicate:VD (const_int 0)))
4452 (vec_concat:<VDBL>
4453 (unspec:VD [(match_dup 1)]
4454 UNSPEC_LD4)
4455 (vec_duplicate:VD (const_int 0))))
4456 (vec_concat:<VRL2>
4457 (vec_concat:<VDBL>
4458 (unspec:VD [(match_dup 1)]
4459 UNSPEC_LD4)
4460 (vec_duplicate:VD (const_int 0)))
4461 (vec_concat:<VDBL>
4462 (unspec:VD [(match_dup 1)]
4463 UNSPEC_LD4)
4464 (vec_duplicate:VD (const_int 0))))) 0))]
4465 "TARGET_SIMD"
4466 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
78ec3036
JG		 4467 [(set_attr "type" "neon_load4_4reg<q>")]
4468)
43e9d192
IB		 4469
4470(define_insn "aarch64_ld4<mode>_dreg"
4471 [(set (match_operand:XI 0 "register_operand" "=w")
4472 (subreg:XI
4473 (vec_concat:<VRL4>
4474 (vec_concat:<VRL2>
4475 (vec_concat:<VDBL>
4476 (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4477 UNSPEC_LD4)
4478 (const_int 0))
4479 (vec_concat:<VDBL>
4480 (unspec:DX [(match_dup 1)]
4481 UNSPEC_LD4)
4482 (const_int 0)))
4483 (vec_concat:<VRL2>
4484 (vec_concat:<VDBL>
4485 (unspec:DX [(match_dup 1)]
4486 UNSPEC_LD4)
4487 (const_int 0))
4488 (vec_concat:<VDBL>
4489 (unspec:DX [(match_dup 1)]
4490 UNSPEC_LD4)
4491 (const_int 0)))) 0))]
4492 "TARGET_SIMD"
4493 "ld1\\t{%S0.1d - %V0.1d}, %1"
78ec3036
JG		 4494 [(set_attr "type" "neon_load1_4reg<q>")]
4495)
43e9d192
IB		 4496
4497(define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4498 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4499 (match_operand:DI 1 "register_operand" "r")
4500 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4501 "TARGET_SIMD"
4502{
ef4bddc2 4503 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
43e9d192
IB		 4504 rtx mem = gen_rtx_MEM (mode, operands[1]);
4505
4506 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4507 DONE;
4508})
4509
dec11868
JG		 4510(define_expand "aarch64_ld1<VALL:mode>"
4511 [(match_operand:VALL 0 "register_operand")
4512 (match_operand:DI 1 "register_operand")]
4513 "TARGET_SIMD"
4514{
ef4bddc2 4515 machine_mode mode = <VALL:MODE>mode;
dec11868 4516 rtx mem = gen_rtx_MEM (mode, operands[1]);
89b4515c
AV		 4517
4518 if (BYTES_BIG_ENDIAN)
4519 emit_insn (gen_aarch64_be_ld1<VALL:mode> (operands[0], mem));
4520 else
4521 emit_move_insn (operands[0], mem);
dec11868
JG		 4522 DONE;
4523})
4524
43e9d192
IB		 4525(define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4526 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4527 (match_operand:DI 1 "register_operand" "r")
4528 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4529 "TARGET_SIMD"
4530{
ef4bddc2 4531 machine_mode mode = <VSTRUCT:MODE>mode;
43e9d192
IB		 4532 rtx mem = gen_rtx_MEM (mode, operands[1]);
4533
4534 emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
4535 DONE;
4536})
4537
3ec1be97
CB		 4538(define_expand "aarch64_ld2_lane<mode>"
4539 [(match_operand:OI 0 "register_operand" "=w")
4540 (match_operand:DI 1 "register_operand" "w")
4541 (match_operand:OI 2 "register_operand" "0")
4542 (match_operand:SI 3 "immediate_operand" "i")
4543 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4544 "TARGET_SIMD"
4545{
ef4bddc2 4546 machine_mode mode = <V_TWO_ELEM>mode;
3ec1be97
CB		 4547 rtx mem = gen_rtx_MEM (mode, operands[1]);
4548
46ed6024
CB		 4549 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4550 NULL);
3ec1be97
CB		 4551 emit_insn (gen_aarch64_vec_load_lanesoi_lane<mode> (operands[0],
4552 mem,
4553 operands[2],
4554 operands[3]));
4555 DONE;
4556})
4557
4558(define_expand "aarch64_ld3_lane<mode>"
4559 [(match_operand:CI 0 "register_operand" "=w")
4560 (match_operand:DI 1 "register_operand" "w")
4561 (match_operand:CI 2 "register_operand" "0")
4562 (match_operand:SI 3 "immediate_operand" "i")
4563 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4564 "TARGET_SIMD"
4565{
ef4bddc2 4566 machine_mode mode = <V_THREE_ELEM>mode;
3ec1be97
CB		 4567 rtx mem = gen_rtx_MEM (mode, operands[1]);
4568
46ed6024
CB		 4569 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4570 NULL);
3ec1be97
CB		 4571 emit_insn (gen_aarch64_vec_load_lanesci_lane<mode> (operands[0],
4572 mem,
4573 operands[2],
4574 operands[3]));
4575 DONE;
4576})
4577
4578(define_expand "aarch64_ld4_lane<mode>"
4579 [(match_operand:XI 0 "register_operand" "=w")
4580 (match_operand:DI 1 "register_operand" "w")
4581 (match_operand:XI 2 "register_operand" "0")
4582 (match_operand:SI 3 "immediate_operand" "i")
4583 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4584 "TARGET_SIMD"
4585{
ef4bddc2 4586 machine_mode mode = <V_FOUR_ELEM>mode;
3ec1be97
CB		 4587 rtx mem = gen_rtx_MEM (mode, operands[1]);
4588
46ed6024
CB		 4589 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4590 NULL);
3ec1be97
CB		 4591 emit_insn (gen_aarch64_vec_load_lanesxi_lane<mode> (operands[0],
4592 mem,
4593 operands[2],
4594 operands[3]));
4595 DONE;
4596})
4597
4598
4599
43e9d192
IB		 4600;; Expanders for builtins to extract vector registers from large
4601;; opaque integer modes.
4602
4603;; D-register list.
4604
4605(define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4606 [(match_operand:VDC 0 "register_operand" "=w")
4607 (match_operand:VSTRUCT 1 "register_operand" "w")
4608 (match_operand:SI 2 "immediate_operand" "i")]
4609 "TARGET_SIMD"
4610{
4611 int part = INTVAL (operands[2]);
4612 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4613 int offset = part * 16;
4614
4615 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4616 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4617 DONE;
4618})
4619
4620;; Q-register list.
4621
4622(define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4623 [(match_operand:VQ 0 "register_operand" "=w")
4624 (match_operand:VSTRUCT 1 "register_operand" "w")
4625 (match_operand:SI 2 "immediate_operand" "i")]
4626 "TARGET_SIMD"
4627{
4628 int part = INTVAL (operands[2]);
4629 int offset = part * 16;
4630
4631 emit_move_insn (operands[0],
4632 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4633 DONE;
4634})
4635
4636;; Permuted-store expanders for neon intrinsics.
4637
88b08073
JG		 4638;; Permute instructions
4639
4640;; vec_perm support
4641
4642(define_expand "vec_perm_const<mode>"
4643 [(match_operand:VALL 0 "register_operand")
4644 (match_operand:VALL 1 "register_operand")
4645 (match_operand:VALL 2 "register_operand")
4646 (match_operand:<V_cmp_result> 3)]
4647 "TARGET_SIMD"
4648{
4649 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4650 operands[2], operands[3]))
4651 DONE;
4652 else
4653 FAIL;
4654})
4655
4656(define_expand "vec_perm<mode>"
4657 [(match_operand:VB 0 "register_operand")
4658 (match_operand:VB 1 "register_operand")
4659 (match_operand:VB 2 "register_operand")
4660 (match_operand:VB 3 "register_operand")]
f7c4e5b8 4661 "TARGET_SIMD"
88b08073
JG		 4662{
4663 aarch64_expand_vec_perm (operands[0], operands[1],
4664 operands[2], operands[3]);
4665 DONE;
4666})
4667
4668(define_insn "aarch64_tbl1<mode>"
4669 [(set (match_operand:VB 0 "register_operand" "=w")
4670 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4671 (match_operand:VB 2 "register_operand" "w")]
4672 UNSPEC_TBL))]
4673 "TARGET_SIMD"
4674 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
78ec3036 4675 [(set_attr "type" "neon_tbl1<q>")]
88b08073
JG		 4676)
4677
4678;; Two source registers.
4679
4680(define_insn "aarch64_tbl2v16qi"
4681 [(set (match_operand:V16QI 0 "register_operand" "=w")
4682 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4683 (match_operand:V16QI 2 "register_operand" "w")]
4684 UNSPEC_TBL))]
4685 "TARGET_SIMD"
4686 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
78ec3036 4687 [(set_attr "type" "neon_tbl2_q")]
88b08073
JG		 4688)
4689
4690(define_insn_and_split "aarch64_combinev16qi"
4691 [(set (match_operand:OI 0 "register_operand" "=w")
4692 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4693 (match_operand:V16QI 2 "register_operand" "w")]
4694 UNSPEC_CONCAT))]
4695 "TARGET_SIMD"
4696 "#"
4697 "&& reload_completed"
4698 [(const_int 0)]
4699{
4700 aarch64_split_combinev16qi (operands);
4701 DONE;
0f686aa9
JG		 4702}
4703[(set_attr "type" "multiple")]
4704)
88b08073 4705
cc4d934f
JG		 4706(define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4707 [(set (match_operand:VALL 0 "register_operand" "=w")
4708 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4709 (match_operand:VALL 2 "register_operand" "w")]
4710 PERMUTE))]
4711 "TARGET_SIMD"
4712 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
78ec3036 4713 [(set_attr "type" "neon_permute<q>")]
cc4d934f
JG		 4714)
4715
ae0533da
AL		 4716;; Note immediate (third) operand is lane index not byte index.
4717(define_insn "aarch64_ext<mode>"
4718 [(set (match_operand:VALL 0 "register_operand" "=w")
4719 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4720 (match_operand:VALL 2 "register_operand" "w")
4721 (match_operand:SI 3 "immediate_operand" "i")]
4722 UNSPEC_EXT))]
4723 "TARGET_SIMD"
4724{
4725 operands[3] = GEN_INT (INTVAL (operands[3])
4726 * GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
4727 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
4728}
4729 [(set_attr "type" "neon_ext<q>")]
4730)
4731
923fcec3
AL		 4732(define_insn "aarch64_rev<REVERSE:rev_op><mode>"
4733 [(set (match_operand:VALL 0 "register_operand" "=w")
4734 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
4735 REVERSE))]
4736 "TARGET_SIMD"
4737 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
4738 [(set_attr "type" "neon_rev<q>")]
4739)
4740
43e9d192
IB		 4741(define_insn "aarch64_st2<mode>_dreg"
4742 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4743 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4744 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4745 UNSPEC_ST2))]
4746 "TARGET_SIMD"
4747 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
78ec3036
JG		 4748 [(set_attr "type" "neon_store2_2reg")]
4749)
43e9d192
IB		 4750
4751(define_insn "aarch64_st2<mode>_dreg"
4752 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4753 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4754 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4755 UNSPEC_ST2))]
4756 "TARGET_SIMD"
4757 "st1\\t{%S1.1d - %T1.1d}, %0"
78ec3036
JG		 4758 [(set_attr "type" "neon_store1_2reg")]
4759)
43e9d192
IB		 4760
4761(define_insn "aarch64_st3<mode>_dreg"
4762 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4763 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4764 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4765 UNSPEC_ST3))]
4766 "TARGET_SIMD"
4767 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
78ec3036
JG		 4768 [(set_attr "type" "neon_store3_3reg")]
4769)
43e9d192
IB		 4770
4771(define_insn "aarch64_st3<mode>_dreg"
4772 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4773 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4774 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4775 UNSPEC_ST3))]
4776 "TARGET_SIMD"
4777 "st1\\t{%S1.1d - %U1.1d}, %0"
78ec3036
JG		 4778 [(set_attr "type" "neon_store1_3reg")]
4779)
43e9d192
IB		 4780
4781(define_insn "aarch64_st4<mode>_dreg"
4782 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4783 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4784 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4785 UNSPEC_ST4))]
4786 "TARGET_SIMD"
4787 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
78ec3036
JG		 4788 [(set_attr "type" "neon_store4_4reg")]
4789)
43e9d192
IB		 4790
4791(define_insn "aarch64_st4<mode>_dreg"
4792 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4793 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4794 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795 UNSPEC_ST4))]
4796 "TARGET_SIMD"
4797 "st1\\t{%S1.1d - %V1.1d}, %0"
78ec3036
JG		 4798 [(set_attr "type" "neon_store1_4reg")]
4799)
43e9d192
IB		 4800
4801(define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4802 [(match_operand:DI 0 "register_operand" "r")
4803 (match_operand:VSTRUCT 1 "register_operand" "w")
4804 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4805 "TARGET_SIMD"
4806{
ef4bddc2 4807 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
43e9d192
IB		 4808 rtx mem = gen_rtx_MEM (mode, operands[0]);
4809
4810 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4811 DONE;
4812})
4813
4814(define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4815 [(match_operand:DI 0 "register_operand" "r")
4816 (match_operand:VSTRUCT 1 "register_operand" "w")
4817 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4818 "TARGET_SIMD"
4819{
ef4bddc2 4820 machine_mode mode = <VSTRUCT:MODE>mode;
43e9d192
IB		 4821 rtx mem = gen_rtx_MEM (mode, operands[0]);
4822
4823 emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4824 DONE;
4825})
4826
ba081b77
JG		 4827(define_expand "aarch64_st2_lane<VQ:mode>"
4828 [(match_operand:DI 0 "register_operand" "r")
4829 (match_operand:OI 1 "register_operand" "w")
4830 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4831 (match_operand:SI 2 "immediate_operand")]
4832 "TARGET_SIMD"
4833{
ef4bddc2 4834 machine_mode mode = <V_TWO_ELEM>mode;
ba081b77
JG		 4835 rtx mem = gen_rtx_MEM (mode, operands[0]);
4836 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4837
4838 emit_insn (gen_vec_store_lanesoi_lane<VQ:mode> (mem,
4839 operands[1],
4840 operands[2]));
4841 DONE;
4842})
4843
4844(define_expand "aarch64_st3_lane<VQ:mode>"
4845 [(match_operand:DI 0 "register_operand" "r")
4846 (match_operand:CI 1 "register_operand" "w")
4847 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4848 (match_operand:SI 2 "immediate_operand")]
4849 "TARGET_SIMD"
4850{
ef4bddc2 4851 machine_mode mode = <V_THREE_ELEM>mode;
ba081b77
JG		 4852 rtx mem = gen_rtx_MEM (mode, operands[0]);
4853 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4854
4855 emit_insn (gen_vec_store_lanesci_lane<VQ:mode> (mem,
4856 operands[1],
4857 operands[2]));
4858 DONE;
4859})
4860
4861(define_expand "aarch64_st4_lane<VQ:mode>"
4862 [(match_operand:DI 0 "register_operand" "r")
4863 (match_operand:XI 1 "register_operand" "w")
4864 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4865 (match_operand:SI 2 "immediate_operand")]
4866 "TARGET_SIMD"
4867{
ef4bddc2 4868 machine_mode mode = <V_FOUR_ELEM>mode;
ba081b77
JG		 4869 rtx mem = gen_rtx_MEM (mode, operands[0]);
4870 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4871
4872 emit_insn (gen_vec_store_lanesxi_lane<VQ:mode> (mem,
4873 operands[1],
4874 operands[2]));
4875 DONE;
4876})
4877
dec11868
JG		 4878(define_expand "aarch64_st1<VALL:mode>"
4879 [(match_operand:DI 0 "register_operand")
4880 (match_operand:VALL 1 "register_operand")]
4881 "TARGET_SIMD"
4882{
ef4bddc2 4883 machine_mode mode = <VALL:MODE>mode;
dec11868 4884 rtx mem = gen_rtx_MEM (mode, operands[0]);
89b4515c
AV		 4885
4886 if (BYTES_BIG_ENDIAN)
4887 emit_insn (gen_aarch64_be_st1<VALL:mode> (mem, operands[1]));
4888 else
4889 emit_move_insn (mem, operands[1]);
dec11868
JG		 4890 DONE;
4891})
4892
43e9d192
IB		 4893;; Expander for builtins to insert vector registers into large
4894;; opaque integer modes.
4895
4896;; Q-register list. We don't need a D-reg inserter as we zero
4897;; extend them in arm_neon.h and insert the resulting Q-regs.
4898
4899(define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4900 [(match_operand:VSTRUCT 0 "register_operand" "+w")
4901 (match_operand:VSTRUCT 1 "register_operand" "0")
4902 (match_operand:VQ 2 "register_operand" "w")
4903 (match_operand:SI 3 "immediate_operand" "i")]
4904 "TARGET_SIMD"
4905{
4906 int part = INTVAL (operands[3]);
4907 int offset = part * 16;
4908
4909 emit_move_insn (operands[0], operands[1]);
4910 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4911 operands[2]);
4912 DONE;
4913})
4914
4369c11e
TB		 4915;; Standard pattern name vec_init<mode>.
4916
4917(define_expand "vec_init<mode>"
4918 [(match_operand:VALL 0 "register_operand" "")
4919 (match_operand 1 "" "")]
4920 "TARGET_SIMD"
4921{
4922 aarch64_expand_vector_init (operands[0], operands[1]);
4923 DONE;
4924})
4925
a50344cb 4926(define_insn "*aarch64_simd_ld1r<mode>"
73295417
FY		 4927 [(set (match_operand:VALL 0 "register_operand" "=w")
4928 (vec_duplicate:VALL
a50344cb
TB		 4929 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4930 "TARGET_SIMD"
4931 "ld1r\\t{%0.<Vtype>}, %1"
78ec3036
JG		 4932 [(set_attr "type" "neon_load1_all_lanes")]
4933)
0050faf8
JG		 4934
4935(define_insn "aarch64_frecpe<mode>"
4936 [(set (match_operand:VDQF 0 "register_operand" "=w")
4937 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4938 UNSPEC_FRECPE))]
4939 "TARGET_SIMD"
4940 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
78ec3036 4941 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
0050faf8
JG		 4942)
4943
fe6f68e2
JG		 4944(define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4945 [(set (match_operand:GPF 0 "register_operand" "=w")
4946 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4947 FRECP))]
4948 "TARGET_SIMD"
4949 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
78ec3036 4950 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
fe6f68e2
JG		 4951)
4952
0050faf8 4953(define_insn "aarch64_frecps<mode>"
fe6f68e2
JG		 4954 [(set (match_operand:VALLF 0 "register_operand" "=w")
4955 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4956 (match_operand:VALLF 2 "register_operand" "w")]
0050faf8
JG		 4957 UNSPEC_FRECPS))]
4958 "TARGET_SIMD"
fe6f68e2 4959 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
78ec3036 4960 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
0050faf8
JG		 4961)
4962
58a3bd25
FY		 4963(define_insn "aarch64_urecpe<mode>"
4964 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
4965 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
4966 UNSPEC_URECPE))]
4967 "TARGET_SIMD"
4968 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
4969 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
4970
0f365c10
TB		 4971;; Standard pattern name vec_extract<mode>.
4972
e58bf20a
TB		 4973(define_expand "vec_extract<mode>"
4974 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
4975 (match_operand:VALL 1 "register_operand" "")
4976 (match_operand:SI 2 "immediate_operand" "")]
0f365c10 4977 "TARGET_SIMD"
e58bf20a
TB		 4978{
4979 emit_insn
4980 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
4981 DONE;
4982})
5a7a4e80
TB		 4983
4984;; aes
4985
4986(define_insn "aarch64_crypto_aes<aes_op>v16qi"
4987 [(set (match_operand:V16QI 0 "register_operand" "=w")
4988 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
4989 (match_operand:V16QI 2 "register_operand" "w")]
4990 CRYPTO_AES))]
4991 "TARGET_SIMD && TARGET_CRYPTO"
4992 "aes<aes_op>\\t%0.16b, %2.16b"
b10baa95 4993 [(set_attr "type" "crypto_aese")]
5a7a4e80
TB		 4994)
4995
4996(define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
4997 [(set (match_operand:V16QI 0 "register_operand" "=w")
4998 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
4999 CRYPTO_AESMC))]
5000 "TARGET_SIMD && TARGET_CRYPTO"
5001 "aes<aesmc_op>\\t%0.16b, %1.16b"
b10baa95 5002 [(set_attr "type" "crypto_aesmc")]
5a7a4e80
TB		 5003)
5004
30442682
TB		 5005;; sha1
5006
5007(define_insn "aarch64_crypto_sha1hsi"
5008 [(set (match_operand:SI 0 "register_operand" "=w")
5009 (unspec:SI [(match_operand:SI 1
5010 "register_operand" "w")]
5011 UNSPEC_SHA1H))]
5012 "TARGET_SIMD && TARGET_CRYPTO"
5013 "sha1h\\t%s0, %s1"
5014 [(set_attr "type" "crypto_sha1_fast")]
5015)
5016
5017(define_insn "aarch64_crypto_sha1su1v4si"
5018 [(set (match_operand:V4SI 0 "register_operand" "=w")
5019 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5020 (match_operand:V4SI 2 "register_operand" "w")]
5021 UNSPEC_SHA1SU1))]
5022 "TARGET_SIMD && TARGET_CRYPTO"
5023 "sha1su1\\t%0.4s, %2.4s"
5024 [(set_attr "type" "crypto_sha1_fast")]
5025)
5026
5027(define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5028 [(set (match_operand:V4SI 0 "register_operand" "=w")
5029 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5030 (match_operand:SI 2 "register_operand" "w")
5031 (match_operand:V4SI 3 "register_operand" "w")]
5032 CRYPTO_SHA1))]
5033 "TARGET_SIMD && TARGET_CRYPTO"
5034 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5035 [(set_attr "type" "crypto_sha1_slow")]
5036)
5037
5038(define_insn "aarch64_crypto_sha1su0v4si"
5039 [(set (match_operand:V4SI 0 "register_operand" "=w")
5040 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5041 (match_operand:V4SI 2 "register_operand" "w")
5042 (match_operand:V4SI 3 "register_operand" "w")]
5043 UNSPEC_SHA1SU0))]
5044 "TARGET_SIMD && TARGET_CRYPTO"
5045 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5046 [(set_attr "type" "crypto_sha1_xor")]
5047)
b9cb0a44
TB		 5048
5049;; sha256
5050
5051(define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5052 [(set (match_operand:V4SI 0 "register_operand" "=w")
5053 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5054 (match_operand:V4SI 2 "register_operand" "w")
5055 (match_operand:V4SI 3 "register_operand" "w")]
5056 CRYPTO_SHA256))]
5057 "TARGET_SIMD && TARGET_CRYPTO"
5058 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5059 [(set_attr "type" "crypto_sha256_slow")]
5060)
5061
5062(define_insn "aarch64_crypto_sha256su0v4si"
5063 [(set (match_operand:V4SI 0 "register_operand" "=w")
5064 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5065 (match_operand:V4SI 2 "register_operand" "w")]
5066 UNSPEC_SHA256SU0))]
5067 "TARGET_SIMD &&TARGET_CRYPTO"
5068 "sha256su0\\t%0.4s, %2.4s"
5069 [(set_attr "type" "crypto_sha256_fast")]
5070)
5071
5072(define_insn "aarch64_crypto_sha256su1v4si"
5073 [(set (match_operand:V4SI 0 "register_operand" "=w")
5074 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5075 (match_operand:V4SI 2 "register_operand" "w")
5076 (match_operand:V4SI 3 "register_operand" "w")]
5077 UNSPEC_SHA256SU1))]
5078 "TARGET_SIMD &&TARGET_CRYPTO"
5079 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5080 [(set_attr "type" "crypto_sha256_slow")]
5081)
7baa225d
TB		 5082
5083;; pmull
5084
5085(define_insn "aarch64_crypto_pmulldi"
5086 [(set (match_operand:TI 0 "register_operand" "=w")
5087 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5088 (match_operand:DI 2 "register_operand" "w")]
5089 UNSPEC_PMULL))]
5090 "TARGET_SIMD && TARGET_CRYPTO"
5091 "pmull\\t%0.1q, %1.1d, %2.1d"
5092 [(set_attr "type" "neon_mul_d_long")]
5093)
5094
5095(define_insn "aarch64_crypto_pmullv2di"
5096 [(set (match_operand:TI 0 "register_operand" "=w")
5097 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5098 (match_operand:V2DI 2 "register_operand" "w")]
5099 UNSPEC_PMULL2))]
5100 "TARGET_SIMD && TARGET_CRYPTO"
5101 "pmull2\\t%0.1q, %1.2d, %2.2d"
5102 [(set_attr "type" "neon_mul_d_long")]
5103)
Mirror of https://gcc.gnu.org/git/gcc.git