]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/config/arm/neon.md
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
2012-09-06 Kyrylo Tkachov <kyrylo.tkachov@arm.com>
[thirdparty/gcc.git] / gcc / config / arm / neon.md
CommitLineData
d98a3884 1;; ARM NEON coprocessor Machine Description
47ddcd6b 2;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012
3;; Free Software Foundation, Inc.
d98a3884 4;; Written by CodeSourcery.
5;;
6;; This file is part of GCC.
7;;
8;; GCC is free software; you can redistribute it and/or modify it
9;; under the terms of the GNU General Public License as published by
038d1e19 10;; the Free Software Foundation; either version 3, or (at your option)
d98a3884 11;; any later version.
12;;
13;; GCC is distributed in the hope that it will be useful, but
14;; WITHOUT ANY WARRANTY; without even the implied warranty of
15;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16;; General Public License for more details.
17;;
18;; You should have received a copy of the GNU General Public License
038d1e19 19;; along with GCC; see the file COPYING3. If not see
20;; <http://www.gnu.org/licenses/>.
d98a3884 21
e7d6d136 22;; Enumerators for unspecs.
23(define_c_enum "unspec" [
24 UNSPEC_ASHIFT_SIGNED
25 UNSPEC_ASHIFT_UNSIGNED
26 UNSPEC_VABD
27 UNSPEC_VABDL
28 UNSPEC_VADD
29 UNSPEC_VADDHN
30 UNSPEC_VADDL
31 UNSPEC_VADDW
32 UNSPEC_VBSL
33 UNSPEC_VCAGE
34 UNSPEC_VCAGT
35 UNSPEC_VCEQ
36 UNSPEC_VCGE
ca6c837f 37 UNSPEC_VCGEU
e7d6d136 38 UNSPEC_VCGT
ca6c837f 39 UNSPEC_VCGTU
e7d6d136 40 UNSPEC_VCLS
47ddcd6b 41 UNSPEC_VCONCAT
e7d6d136 42 UNSPEC_VCVT
43 UNSPEC_VCVT_N
44 UNSPEC_VEXT
45 UNSPEC_VHADD
46 UNSPEC_VHSUB
47 UNSPEC_VLD1
e7d6d136 48 UNSPEC_VLD1_LANE
49 UNSPEC_VLD2
50 UNSPEC_VLD2_DUP
51 UNSPEC_VLD2_LANE
52 UNSPEC_VLD3
53 UNSPEC_VLD3A
54 UNSPEC_VLD3B
55 UNSPEC_VLD3_DUP
56 UNSPEC_VLD3_LANE
57 UNSPEC_VLD4
58 UNSPEC_VLD4A
59 UNSPEC_VLD4B
60 UNSPEC_VLD4_DUP
61 UNSPEC_VLD4_LANE
62 UNSPEC_VMAX
63 UNSPEC_VMIN
64 UNSPEC_VMLA
65 UNSPEC_VMLAL
66 UNSPEC_VMLA_LANE
67 UNSPEC_VMLAL_LANE
68 UNSPEC_VMLS
69 UNSPEC_VMLSL
70 UNSPEC_VMLS_LANE
71 UNSPEC_VMLSL_LANE
72 UNSPEC_VMOVL
73 UNSPEC_VMOVN
74 UNSPEC_VMUL
75 UNSPEC_VMULL
76 UNSPEC_VMUL_LANE
77 UNSPEC_VMULL_LANE
78 UNSPEC_VPADAL
79 UNSPEC_VPADD
80 UNSPEC_VPADDL
81 UNSPEC_VPMAX
82 UNSPEC_VPMIN
83 UNSPEC_VPSMAX
84 UNSPEC_VPSMIN
85 UNSPEC_VPUMAX
86 UNSPEC_VPUMIN
87 UNSPEC_VQABS
88 UNSPEC_VQADD
89 UNSPEC_VQDMLAL
90 UNSPEC_VQDMLAL_LANE
91 UNSPEC_VQDMLSL
92 UNSPEC_VQDMLSL_LANE
93 UNSPEC_VQDMULH
94 UNSPEC_VQDMULH_LANE
95 UNSPEC_VQDMULL
96 UNSPEC_VQDMULL_LANE
97 UNSPEC_VQMOVN
98 UNSPEC_VQMOVUN
99 UNSPEC_VQNEG
100 UNSPEC_VQSHL
101 UNSPEC_VQSHL_N
102 UNSPEC_VQSHLU_N
103 UNSPEC_VQSHRN_N
104 UNSPEC_VQSHRUN_N
105 UNSPEC_VQSUB
106 UNSPEC_VRECPE
107 UNSPEC_VRECPS
108 UNSPEC_VREV16
109 UNSPEC_VREV32
110 UNSPEC_VREV64
111 UNSPEC_VRSQRTE
112 UNSPEC_VRSQRTS
113 UNSPEC_VSHL
114 UNSPEC_VSHLL_N
115 UNSPEC_VSHL_N
116 UNSPEC_VSHR_N
117 UNSPEC_VSHRN_N
118 UNSPEC_VSLI
119 UNSPEC_VSRA_N
120 UNSPEC_VSRI
121 UNSPEC_VST1
122 UNSPEC_VST1_LANE
123 UNSPEC_VST2
124 UNSPEC_VST2_LANE
125 UNSPEC_VST3
126 UNSPEC_VST3A
127 UNSPEC_VST3B
128 UNSPEC_VST3_LANE
129 UNSPEC_VST4
130 UNSPEC_VST4A
131 UNSPEC_VST4B
132 UNSPEC_VST4_LANE
133 UNSPEC_VSTRUCTDUMMY
134 UNSPEC_VSUB
135 UNSPEC_VSUBHN
136 UNSPEC_VSUBL
137 UNSPEC_VSUBW
138 UNSPEC_VTBL
139 UNSPEC_VTBX
140 UNSPEC_VTRN1
141 UNSPEC_VTRN2
142 UNSPEC_VTST
143 UNSPEC_VUZP1
144 UNSPEC_VUZP2
145 UNSPEC_VZIP1
146 UNSPEC_VZIP2
147 UNSPEC_MISALIGNED_ACCESS
148 UNSPEC_VCLE
149 UNSPEC_VCLT
150])
d98a3884 151
d98a3884 152
bcaec148 153;; Attribute used to permit string comparisons against <VQH_mnem> in
154;; neon_type attribute definitions.
155(define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
156
d98a3884 157(define_insn "*neon_mov<mode>"
87f800b2 158 [(set (match_operand:VDX 0 "nonimmediate_operand"
d98a3884 159 "=w,Uv,w, w, ?r,?w,?r,?r, ?Us")
87f800b2 160 (match_operand:VDX 1 "general_operand"
d98a3884 161 " w,w, Dn,Uvi, w, r, r, Usi,r"))]
fdd8cbd8 162 "TARGET_NEON
163 && (register_operand (operands[0], <MODE>mode)
164 || register_operand (operands[1], <MODE>mode))"
d98a3884 165{
166 if (which_alternative == 2)
167 {
168 int width, is_valid;
169 static char templ[40];
170
171 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
172 &operands[1], &width);
173
174 gcc_assert (is_valid != 0);
175
176 if (width == 0)
177 return "vmov.f32\t%P0, %1 @ <mode>";
178 else
87f800b2 179 sprintf (templ, "vmov.i%d\t%%P0, %%x1 @ <mode>", width);
d98a3884 180
181 return templ;
182 }
183
184 /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp
185 below must be changed to output_move_neon (which will use the
f1c02c87 186 element/structure loads/stores), and the constraint changed to 'Um' instead
d98a3884 187 of 'Uv'. */
188
189 switch (which_alternative)
190 {
191 case 0: return "vmov\t%P0, %P1 @ <mode>";
192 case 1: case 3: return output_move_vfp (operands);
193 case 2: gcc_unreachable ();
194 case 4: return "vmov\t%Q0, %R0, %P1 @ <mode>";
195 case 5: return "vmov\t%P0, %Q1, %R1 @ <mode>";
26ff80c0 196 default: return output_move_double (operands, true, NULL);
d98a3884 197 }
198}
bcaec148 199 [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
200 (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
201 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
202 (set_attr "length" "4,4,4,4,4,4,8,8,8")
203 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
8848d797 204 (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
d98a3884 205
206(define_insn "*neon_mov<mode>"
207 [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
208 "=w,Un,w, w, ?r,?w,?r,?r, ?Us")
209 (match_operand:VQXMOV 1 "general_operand"
210 " w,w, Dn,Uni, w, r, r, Usi, r"))]
fdd8cbd8 211 "TARGET_NEON
212 && (register_operand (operands[0], <MODE>mode)
213 || register_operand (operands[1], <MODE>mode))"
d98a3884 214{
215 if (which_alternative == 2)
216 {
217 int width, is_valid;
218 static char templ[40];
219
220 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
221 &operands[1], &width);
222
223 gcc_assert (is_valid != 0);
224
225 if (width == 0)
226 return "vmov.f32\t%q0, %1 @ <mode>";
227 else
228 sprintf (templ, "vmov.i%d\t%%q0, %%1 @ <mode>", width);
229
230 return templ;
231 }
232
233 switch (which_alternative)
234 {
235 case 0: return "vmov\t%q0, %q1 @ <mode>";
236 case 1: case 3: return output_move_neon (operands);
237 case 2: gcc_unreachable ();
238 case 4: return "vmov\t%Q0, %R0, %e1 @ <mode>\;vmov\t%J0, %K0, %f1";
239 case 5: return "vmov\t%e0, %Q1, %R1 @ <mode>\;vmov\t%f0, %J1, %K1";
240 default: return output_move_quad (operands);
241 }
242}
bcaec148 243 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
244 neon_mrrc,neon_mcr_2_mcrr,*,*,*")
245 (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
246 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
d98a3884 247 (set_attr "length" "4,8,4,8,8,8,16,8,16")
248 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
8848d797 249 (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
d98a3884 250
251(define_expand "movti"
252 [(set (match_operand:TI 0 "nonimmediate_operand" "")
253 (match_operand:TI 1 "general_operand" ""))]
254 "TARGET_NEON"
255{
fdd8cbd8 256 if (can_create_pseudo_p ())
257 {
0438d37f 258 if (!REG_P (operands[0]))
fdd8cbd8 259 operands[1] = force_reg (TImode, operands[1]);
260 }
d98a3884 261})
262
263(define_expand "mov<mode>"
264 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
265 (match_operand:VSTRUCT 1 "general_operand" ""))]
266 "TARGET_NEON"
267{
fdd8cbd8 268 if (can_create_pseudo_p ())
269 {
0438d37f 270 if (!REG_P (operands[0]))
fdd8cbd8 271 operands[1] = force_reg (<MODE>mode, operands[1]);
272 }
d98a3884 273})
274
275(define_insn "*neon_mov<mode>"
276 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
277 (match_operand:VSTRUCT 1 "general_operand" " w,w, Ut"))]
fdd8cbd8 278 "TARGET_NEON
279 && (register_operand (operands[0], <MODE>mode)
280 || register_operand (operands[1], <MODE>mode))"
d98a3884 281{
282 switch (which_alternative)
283 {
284 case 0: return "#";
285 case 1: case 2: return output_move_neon (operands);
286 default: gcc_unreachable ();
287 }
288}
c52acdd2 289 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
ba6c018a 290 (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
d98a3884 291
292(define_split
293 [(set (match_operand:EI 0 "s_register_operand" "")
294 (match_operand:EI 1 "s_register_operand" ""))]
295 "TARGET_NEON && reload_completed"
296 [(set (match_dup 0) (match_dup 1))
297 (set (match_dup 2) (match_dup 3))]
298{
299 int rdest = REGNO (operands[0]);
300 int rsrc = REGNO (operands[1]);
301 rtx dest[2], src[2];
302
303 dest[0] = gen_rtx_REG (TImode, rdest);
304 src[0] = gen_rtx_REG (TImode, rsrc);
305 dest[1] = gen_rtx_REG (DImode, rdest + 4);
306 src[1] = gen_rtx_REG (DImode, rsrc + 4);
307
308 neon_disambiguate_copy (operands, dest, src, 2);
309})
310
311(define_split
312 [(set (match_operand:OI 0 "s_register_operand" "")
313 (match_operand:OI 1 "s_register_operand" ""))]
314 "TARGET_NEON && reload_completed"
315 [(set (match_dup 0) (match_dup 1))
316 (set (match_dup 2) (match_dup 3))]
317{
318 int rdest = REGNO (operands[0]);
319 int rsrc = REGNO (operands[1]);
320 rtx dest[2], src[2];
321
322 dest[0] = gen_rtx_REG (TImode, rdest);
323 src[0] = gen_rtx_REG (TImode, rsrc);
324 dest[1] = gen_rtx_REG (TImode, rdest + 4);
325 src[1] = gen_rtx_REG (TImode, rsrc + 4);
326
327 neon_disambiguate_copy (operands, dest, src, 2);
328})
329
330(define_split
331 [(set (match_operand:CI 0 "s_register_operand" "")
332 (match_operand:CI 1 "s_register_operand" ""))]
333 "TARGET_NEON && reload_completed"
334 [(set (match_dup 0) (match_dup 1))
335 (set (match_dup 2) (match_dup 3))
336 (set (match_dup 4) (match_dup 5))]
337{
338 int rdest = REGNO (operands[0]);
339 int rsrc = REGNO (operands[1]);
340 rtx dest[3], src[3];
341
342 dest[0] = gen_rtx_REG (TImode, rdest);
343 src[0] = gen_rtx_REG (TImode, rsrc);
344 dest[1] = gen_rtx_REG (TImode, rdest + 4);
345 src[1] = gen_rtx_REG (TImode, rsrc + 4);
346 dest[2] = gen_rtx_REG (TImode, rdest + 8);
347 src[2] = gen_rtx_REG (TImode, rsrc + 8);
348
349 neon_disambiguate_copy (operands, dest, src, 3);
350})
351
352(define_split
353 [(set (match_operand:XI 0 "s_register_operand" "")
354 (match_operand:XI 1 "s_register_operand" ""))]
355 "TARGET_NEON && reload_completed"
356 [(set (match_dup 0) (match_dup 1))
357 (set (match_dup 2) (match_dup 3))
358 (set (match_dup 4) (match_dup 5))
359 (set (match_dup 6) (match_dup 7))]
360{
361 int rdest = REGNO (operands[0]);
362 int rsrc = REGNO (operands[1]);
363 rtx dest[4], src[4];
364
365 dest[0] = gen_rtx_REG (TImode, rdest);
366 src[0] = gen_rtx_REG (TImode, rsrc);
367 dest[1] = gen_rtx_REG (TImode, rdest + 4);
368 src[1] = gen_rtx_REG (TImode, rsrc + 4);
369 dest[2] = gen_rtx_REG (TImode, rdest + 8);
370 src[2] = gen_rtx_REG (TImode, rsrc + 8);
371 dest[3] = gen_rtx_REG (TImode, rdest + 12);
372 src[3] = gen_rtx_REG (TImode, rsrc + 12);
373
374 neon_disambiguate_copy (operands, dest, src, 4);
375})
376
dd0cb1e8 377(define_expand "movmisalign<mode>"
4c1dfb84 378 [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
379 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
dd0cb1e8 380 UNSPEC_MISALIGNED_ACCESS))]
381 "TARGET_NEON && !BYTES_BIG_ENDIAN"
382{
383 /* This pattern is not permitted to fail during expansion: if both arguments
384 are non-registers (e.g. memory := constant, which can be created by the
385 auto-vectorizer), force operand 1 into a register. */
386 if (!s_register_operand (operands[0], <MODE>mode)
387 && !s_register_operand (operands[1], <MODE>mode))
388 operands[1] = force_reg (<MODE>mode, operands[1]);
389})
390
391(define_insn "*movmisalign<mode>_neon_store"
4c1dfb84 392 [(set (match_operand:VDX 0 "neon_struct_operand" "=Um")
dd0cb1e8 393 (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
394 UNSPEC_MISALIGNED_ACCESS))]
395 "TARGET_NEON && !BYTES_BIG_ENDIAN"
396 "vst1.<V_sz_elem>\t{%P1}, %A0"
397 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
398
399(define_insn "*movmisalign<mode>_neon_load"
4c1dfb84 400 [(set (match_operand:VDX 0 "s_register_operand" "=w")
401 (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
dd0cb1e8 402 UNSPEC_MISALIGNED_ACCESS))]
403 "TARGET_NEON && !BYTES_BIG_ENDIAN"
404 "vld1.<V_sz_elem>\t{%P0}, %A1"
405 [(set_attr "neon_type" "neon_vld1_1_2_regs")])
406
407(define_insn "*movmisalign<mode>_neon_store"
4c1dfb84 408 [(set (match_operand:VQX 0 "neon_struct_operand" "=Um")
dd0cb1e8 409 (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
410 UNSPEC_MISALIGNED_ACCESS))]
411 "TARGET_NEON && !BYTES_BIG_ENDIAN"
412 "vst1.<V_sz_elem>\t{%q1}, %A0"
413 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
414
415(define_insn "*movmisalign<mode>_neon_load"
4c1dfb84 416 [(set (match_operand:VQX 0 "s_register_operand" "=w")
417 (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
dd0cb1e8 418 UNSPEC_MISALIGNED_ACCESS))]
419 "TARGET_NEON && !BYTES_BIG_ENDIAN"
420 "vld1.<V_sz_elem>\t{%q0}, %A1"
421 [(set_attr "neon_type" "neon_vld1_1_2_regs")])
422
eecb5747 423(define_insn "vec_set<mode>_internal"
424 [(set (match_operand:VD 0 "s_register_operand" "=w")
d98a3884 425 (vec_merge:VD
d98a3884 426 (vec_duplicate:VD
427 (match_operand:<V_elem> 1 "s_register_operand" "r"))
eecb5747 428 (match_operand:VD 3 "s_register_operand" "0")
429 (match_operand:SI 2 "immediate_operand" "i")))]
d98a3884 430 "TARGET_NEON"
eecb5747 431{
e3c4ab51 432 int elt = ffs ((int) INTVAL (operands[2])) - 1;
cdf93281 433 if (BYTES_BIG_ENDIAN)
434 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
435 operands[2] = GEN_INT (elt);
eecb5747 436
17ae1a66 437 return "vmov.<V_sz_elem>\t%P0[%c2], %1";
eecb5747 438}
17ae1a66 439 [(set_attr "neon_type" "neon_mcr")])
d98a3884 440
eecb5747 441(define_insn "vec_set<mode>_internal"
442 [(set (match_operand:VQ 0 "s_register_operand" "=w")
d98a3884 443 (vec_merge:VQ
d98a3884 444 (vec_duplicate:VQ
445 (match_operand:<V_elem> 1 "s_register_operand" "r"))
eecb5747 446 (match_operand:VQ 3 "s_register_operand" "0")
447 (match_operand:SI 2 "immediate_operand" "i")))]
d98a3884 448 "TARGET_NEON"
449{
d3c9aa05 450 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
d98a3884 451 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
eecb5747 452 int elt = elem % half_elts;
453 int hi = (elem / half_elts) * 2;
d98a3884 454 int regno = REGNO (operands[0]);
455
cdf93281 456 if (BYTES_BIG_ENDIAN)
457 elt = half_elts - 1 - elt;
458
d98a3884 459 operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
460 operands[2] = GEN_INT (elt);
461
17ae1a66 462 return "vmov.<V_sz_elem>\t%P0[%c2], %1";
d98a3884 463}
17ae1a66 464 [(set_attr "neon_type" "neon_mcr")]
bcaec148 465)
d98a3884 466
eecb5747 467(define_insn "vec_setv2di_internal"
468 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
d98a3884 469 (vec_merge:V2DI
d98a3884 470 (vec_duplicate:V2DI
471 (match_operand:DI 1 "s_register_operand" "r"))
eecb5747 472 (match_operand:V2DI 3 "s_register_operand" "0")
473 (match_operand:SI 2 "immediate_operand" "i")))]
d98a3884 474 "TARGET_NEON"
475{
d3c9aa05 476 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
eecb5747 477 int regno = REGNO (operands[0]) + 2 * elem;
d98a3884 478
479 operands[0] = gen_rtx_REG (DImode, regno);
480
17ae1a66 481 return "vmov\t%P0, %Q1, %R1";
d98a3884 482}
17ae1a66 483 [(set_attr "neon_type" "neon_mcr_2_mcrr")]
bcaec148 484)
d98a3884 485
eecb5747 486(define_expand "vec_set<mode>"
487 [(match_operand:VDQ 0 "s_register_operand" "")
488 (match_operand:<V_elem> 1 "s_register_operand" "")
489 (match_operand:SI 2 "immediate_operand" "")]
490 "TARGET_NEON"
491{
492 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
493 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
494 GEN_INT (elem), operands[0]));
495 DONE;
496})
497
d98a3884 498(define_insn "vec_extract<mode>"
499 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
500 (vec_select:<V_elem>
501 (match_operand:VD 1 "s_register_operand" "w")
502 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
503 "TARGET_NEON"
cdf93281 504{
505 if (BYTES_BIG_ENDIAN)
506 {
507 int elt = INTVAL (operands[2]);
508 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
509 operands[2] = GEN_INT (elt);
510 }
17ae1a66 511 return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
cdf93281 512}
17ae1a66 513 [(set_attr "neon_type" "neon_bp_simple")]
bcaec148 514)
d98a3884 515
516(define_insn "vec_extract<mode>"
517 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
518 (vec_select:<V_elem>
519 (match_operand:VQ 1 "s_register_operand" "w")
520 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
521 "TARGET_NEON"
522{
523 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
524 int elt = INTVAL (operands[2]) % half_elts;
525 int hi = (INTVAL (operands[2]) / half_elts) * 2;
526 int regno = REGNO (operands[1]);
527
cdf93281 528 if (BYTES_BIG_ENDIAN)
529 elt = half_elts - 1 - elt;
530
d98a3884 531 operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
532 operands[2] = GEN_INT (elt);
533
17ae1a66 534 return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
d98a3884 535}
17ae1a66 536 [(set_attr "neon_type" "neon_bp_simple")]
bcaec148 537)
d98a3884 538
539(define_insn "vec_extractv2di"
540 [(set (match_operand:DI 0 "s_register_operand" "=r")
541 (vec_select:DI
542 (match_operand:V2DI 1 "s_register_operand" "w")
543 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
544 "TARGET_NEON"
545{
d0f1d3d4 546 int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
d98a3884 547
548 operands[1] = gen_rtx_REG (DImode, regno);
549
17ae1a66 550 return "vmov\t%Q0, %R0, %P1 @ v2di";
d98a3884 551}
17ae1a66 552 [(set_attr "neon_type" "neon_int_1")]
bcaec148 553)
d98a3884 554
555(define_expand "vec_init<mode>"
556 [(match_operand:VDQ 0 "s_register_operand" "")
557 (match_operand 1 "" "")]
558 "TARGET_NEON"
559{
560 neon_expand_vector_init (operands[0], operands[1]);
561 DONE;
562})
563
564;; Doubleword and quadword arithmetic.
565
94829feb 566;; NOTE: some other instructions also support 64-bit integer
567;; element size, which we could potentially use for "long long" operations.
d98a3884 568
569(define_insn "*add<mode>3_neon"
570 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
571 (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
572 (match_operand:VDQ 2 "s_register_operand" "w")))]
fe4048be 573 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
bcaec148 574 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
575 [(set (attr "neon_type")
0bf497f5 576 (if_then_else (match_test "<Is_float_mode>")
577 (if_then_else (match_test "<Is_d_reg>")
bcaec148 578 (const_string "neon_fp_vadd_ddd_vabs_dd")
579 (const_string "neon_fp_vadd_qqq_vabs_qq"))
580 (const_string "neon_int_1")))]
581)
d98a3884 582
94829feb 583(define_insn "adddi3_neon"
10e5ccd5 584 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
585 (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
586 (match_operand:DI 2 "arm_adddi_operand" "w,r,0,w,r,Dd,Dd")))
94829feb 587 (clobber (reg:CC CC_REGNUM))]
588 "TARGET_NEON"
589{
590 switch (which_alternative)
591 {
a651f34d 592 case 0: /* fall through */
593 case 3: return "vadd.i64\t%P0, %P1, %P2";
94829feb 594 case 1: return "#";
595 case 2: return "#";
10e5ccd5 596 case 4: return "#";
597 case 5: return "#";
598 case 6: return "#";
94829feb 599 default: gcc_unreachable ();
600 }
601}
10e5ccd5 602 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
603 (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
604 (set_attr "length" "*,8,8,*,8,8,8")
605 (set_attr "arch" "nota8,*,*,onlya8,*,*,*")]
94829feb 606)
607
d98a3884 608(define_insn "*sub<mode>3_neon"
609 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
610 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
611 (match_operand:VDQ 2 "s_register_operand" "w")))]
fe4048be 612 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
bcaec148 613 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
614 [(set (attr "neon_type")
0bf497f5 615 (if_then_else (match_test "<Is_float_mode>")
616 (if_then_else (match_test "<Is_d_reg>")
bcaec148 617 (const_string "neon_fp_vadd_ddd_vabs_dd")
618 (const_string "neon_fp_vadd_qqq_vabs_qq"))
619 (const_string "neon_int_2")))]
620)
d98a3884 621
94829feb 622(define_insn "subdi3_neon"
a651f34d 623 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
624 (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
625 (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
94829feb 626 (clobber (reg:CC CC_REGNUM))]
627 "TARGET_NEON"
628{
629 switch (which_alternative)
630 {
a651f34d 631 case 0: /* fall through */
632 case 4: return "vsub.i64\t%P0, %P1, %P2";
94829feb 633 case 1: /* fall through */
634 case 2: /* fall through */
635 case 3: return "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
636 default: gcc_unreachable ();
637 }
638}
a651f34d 639 [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
640 (set_attr "conds" "*,clob,clob,clob,*")
641 (set_attr "length" "*,8,8,8,*")
642 (set_attr "arch" "nota8,*,*,*,onlya8")]
94829feb 643)
644
d98a3884 645(define_insn "*mul<mode>3_neon"
646 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
647 (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
648 (match_operand:VDQ 2 "s_register_operand" "w")))]
fe4048be 649 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
bcaec148 650 "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
651 [(set (attr "neon_type")
0bf497f5 652 (if_then_else (match_test "<Is_float_mode>")
653 (if_then_else (match_test "<Is_d_reg>")
bcaec148 654 (const_string "neon_fp_vadd_ddd_vabs_dd")
655 (const_string "neon_fp_vadd_qqq_vabs_qq"))
0bf497f5 656 (if_then_else (match_test "<Is_d_reg>")
bcaec148 657 (if_then_else
0bf497f5 658 (match_test "<Scalar_mul_8_16>")
bcaec148 659 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
660 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
0bf497f5 661 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 662 (const_string "neon_mul_qqq_8_16_32_ddd_32")
663 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
664)
d98a3884 665
94829feb 666(define_insn "mul<mode>3add<mode>_neon"
3d8bad71 667 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
668 (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
669 (match_operand:VDQ 3 "s_register_operand" "w"))
670 (match_operand:VDQ 1 "s_register_operand" "0")))]
fe4048be 671 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
3d8bad71 672 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
673 [(set (attr "neon_type")
0bf497f5 674 (if_then_else (match_test "<Is_float_mode>")
675 (if_then_else (match_test "<Is_d_reg>")
3d8bad71 676 (const_string "neon_fp_vmla_ddd")
677 (const_string "neon_fp_vmla_qqq"))
0bf497f5 678 (if_then_else (match_test "<Is_d_reg>")
3d8bad71 679 (if_then_else
0bf497f5 680 (match_test "<Scalar_mul_8_16>")
3d8bad71 681 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
682 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
0bf497f5 683 (if_then_else (match_test "<Scalar_mul_8_16>")
3d8bad71 684 (const_string "neon_mla_qqq_8_16")
685 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
686)
687
94829feb 688(define_insn "mul<mode>3neg<mode>add<mode>_neon"
3d8bad71 689 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
690 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
691 (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
692 (match_operand:VDQ 3 "s_register_operand" "w"))))]
fe4048be 693 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
3d8bad71 694 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
695 [(set (attr "neon_type")
0bf497f5 696 (if_then_else (match_test "<Is_float_mode>")
697 (if_then_else (match_test "<Is_d_reg>")
3d8bad71 698 (const_string "neon_fp_vmla_ddd")
699 (const_string "neon_fp_vmla_qqq"))
0bf497f5 700 (if_then_else (match_test "<Is_d_reg>")
3d8bad71 701 (if_then_else
0bf497f5 702 (match_test "<Scalar_mul_8_16>")
3d8bad71 703 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
704 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
0bf497f5 705 (if_then_else (match_test "<Scalar_mul_8_16>")
3d8bad71 706 (const_string "neon_mla_qqq_8_16")
707 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
708)
709
d98a3884 710(define_insn "ior<mode>3"
711 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
712 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
713 (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
714 "TARGET_NEON"
715{
716 switch (which_alternative)
717 {
718 case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
719 case 1: return neon_output_logic_immediate ("vorr", &operands[2],
720 <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
721 default: gcc_unreachable ();
722 }
bcaec148 723}
724 [(set_attr "neon_type" "neon_int_1")]
725)
d98a3884 726
727(define_insn "iordi3_neon"
a651f34d 728 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
729 (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
730 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
d98a3884 731 "TARGET_NEON"
732{
733 switch (which_alternative)
734 {
a651f34d 735 case 0: /* fall through */
736 case 4: return "vorr\t%P0, %P1, %P2";
737 case 1: /* fall through */
738 case 5: return neon_output_logic_immediate ("vorr", &operands[2],
d98a3884 739 DImode, 0, VALID_NEON_QREG_MODE (DImode));
e2669ea7 740 case 2: return "#";
741 case 3: return "#";
d98a3884 742 default: gcc_unreachable ();
743 }
bcaec148 744}
a651f34d 745 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
746 (set_attr "length" "*,*,8,8,*,*")
747 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
bcaec148 748)
d98a3884 749
750;; The concrete forms of the Neon immediate-logic instructions are vbic and
751;; vorr. We support the pseudo-instruction vand instead, because that
752;; corresponds to the canonical form the middle-end expects to use for
753;; immediate bitwise-ANDs.
754
755(define_insn "and<mode>3"
756 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
757 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
758 (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
759 "TARGET_NEON"
760{
761 switch (which_alternative)
762 {
763 case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
764 case 1: return neon_output_logic_immediate ("vand", &operands[2],
765 <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
766 default: gcc_unreachable ();
767 }
bcaec148 768}
769 [(set_attr "neon_type" "neon_int_1")]
770)
d98a3884 771
772(define_insn "anddi3_neon"
a651f34d 773 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
774 (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
775 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
d98a3884 776 "TARGET_NEON"
777{
778 switch (which_alternative)
779 {
a651f34d 780 case 0: /* fall through */
781 case 4: return "vand\t%P0, %P1, %P2";
782 case 1: /* fall through */
783 case 5: return neon_output_logic_immediate ("vand", &operands[2],
d98a3884 784 DImode, 1, VALID_NEON_QREG_MODE (DImode));
e2669ea7 785 case 2: return "#";
786 case 3: return "#";
d98a3884 787 default: gcc_unreachable ();
788 }
bcaec148 789}
a651f34d 790 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
791 (set_attr "length" "*,*,8,8,*,*")
792 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
bcaec148 793)
d98a3884 794
795(define_insn "orn<mode>3_neon"
796 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
04931b44 797 (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
798 (match_operand:VDQ 1 "s_register_operand" "w")))]
d98a3884 799 "TARGET_NEON"
bcaec148 800 "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
801 [(set_attr "neon_type" "neon_int_1")]
802)
d98a3884 803
2eb9782e 804;; TODO: investigate whether we should disable
805;; this and bicdi3_neon for the A8 in line with the other
806;; changes above.
807(define_insn_and_split "orndi3_neon"
808 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
809 (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
810 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
d98a3884 811 "TARGET_NEON"
e2669ea7 812 "@
813 vorn\t%P0, %P1, %P2
814 #
2eb9782e 815 #
e2669ea7 816 #"
2eb9782e 817 "reload_completed &&
818 (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
819 [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
820 (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
821 "
822 {
823 if (TARGET_THUMB2)
824 {
825 operands[3] = gen_highpart (SImode, operands[0]);
826 operands[0] = gen_lowpart (SImode, operands[0]);
827 operands[4] = gen_highpart (SImode, operands[2]);
828 operands[2] = gen_lowpart (SImode, operands[2]);
829 operands[5] = gen_highpart (SImode, operands[1]);
830 operands[1] = gen_lowpart (SImode, operands[1]);
831 }
832 else
833 {
834 emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
835 emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
836 DONE;
837 }
838 }"
839 [(set_attr "neon_type" "neon_int_1,*,*,*")
840 (set_attr "length" "*,16,8,8")
841 (set_attr "arch" "any,a,t2,t2")]
bcaec148 842)
d98a3884 843
844(define_insn "bic<mode>3_neon"
845 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
04931b44 846 (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
847 (match_operand:VDQ 1 "s_register_operand" "w")))]
d98a3884 848 "TARGET_NEON"
bcaec148 849 "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
850 [(set_attr "neon_type" "neon_int_1")]
851)
d98a3884 852
e2669ea7 853;; Compare to *anddi_notdi_di.
d98a3884 854(define_insn "bicdi3_neon"
e2669ea7 855 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
856 (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
857 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
d98a3884 858 "TARGET_NEON"
e2669ea7 859 "@
860 vbic\t%P0, %P1, %P2
861 #
862 #"
863 [(set_attr "neon_type" "neon_int_1,*,*")
864 (set_attr "length" "*,8,8")]
bcaec148 865)
d98a3884 866
867(define_insn "xor<mode>3"
868 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
869 (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
870 (match_operand:VDQ 2 "s_register_operand" "w")))]
871 "TARGET_NEON"
bcaec148 872 "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
873 [(set_attr "neon_type" "neon_int_1")]
874)
d98a3884 875
876(define_insn "xordi3_neon"
a651f34d 877 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
878 (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
879 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
d98a3884 880 "TARGET_NEON"
e2669ea7 881 "@
882 veor\t%P0, %P1, %P2
883 #
a651f34d 884 #
885 veor\t%P0, %P1, %P2"
886 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
887 (set_attr "length" "*,8,8,*")
888 (set_attr "arch" "nota8,*,*,onlya8")]
bcaec148 889)
d98a3884 890
891(define_insn "one_cmpl<mode>2"
892 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
893 (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
894 "TARGET_NEON"
bcaec148 895 "vmvn\t%<V_reg>0, %<V_reg>1"
896 [(set_attr "neon_type" "neon_int_1")]
897)
d98a3884 898
899(define_insn "abs<mode>2"
900 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
901 (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
902 "TARGET_NEON"
bcaec148 903 "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
904 [(set (attr "neon_type")
0bf497f5 905 (if_then_else (match_test "<Is_float_mode>")
906 (if_then_else (match_test "<Is_d_reg>")
bcaec148 907 (const_string "neon_fp_vadd_ddd_vabs_dd")
908 (const_string "neon_fp_vadd_qqq_vabs_qq"))
909 (const_string "neon_int_3")))]
910)
d98a3884 911
912(define_insn "neg<mode>2"
913 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
914 (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
915 "TARGET_NEON"
bcaec148 916 "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
917 [(set (attr "neon_type")
0bf497f5 918 (if_then_else (match_test "<Is_float_mode>")
919 (if_then_else (match_test "<Is_d_reg>")
bcaec148 920 (const_string "neon_fp_vadd_ddd_vabs_dd")
921 (const_string "neon_fp_vadd_qqq_vabs_qq"))
922 (const_string "neon_int_3")))]
923)
d98a3884 924
774d2fbb 925(define_insn "negdi2_neon"
926 [(set (match_operand:DI 0 "s_register_operand" "=&w, w,r,&r")
927 (neg:DI (match_operand:DI 1 "s_register_operand" " w, w,0, r")))
928 (clobber (match_scratch:DI 2 "= X,&w,X, X"))
929 (clobber (reg:CC CC_REGNUM))]
930 "TARGET_NEON"
931 "#"
932 [(set_attr "length" "8")]
933)
934
935; Split negdi2_neon for vfp registers
936(define_split
937 [(set (match_operand:DI 0 "s_register_operand" "")
938 (neg:DI (match_operand:DI 1 "s_register_operand" "")))
939 (clobber (match_scratch:DI 2 ""))
940 (clobber (reg:CC CC_REGNUM))]
941 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
942 [(set (match_dup 2) (const_int 0))
943 (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
944 (clobber (reg:CC CC_REGNUM))])]
945 {
946 if (!REG_P (operands[2]))
947 operands[2] = operands[0];
948 }
949)
950
951; Split negdi2_neon for core registers
952(define_split
953 [(set (match_operand:DI 0 "s_register_operand" "")
954 (neg:DI (match_operand:DI 1 "s_register_operand" "")))
955 (clobber (match_scratch:DI 2 ""))
956 (clobber (reg:CC CC_REGNUM))]
957 "TARGET_32BIT && reload_completed
958 && arm_general_register_operand (operands[0], DImode)"
959 [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
960 (clobber (reg:CC CC_REGNUM))])]
961 ""
962)
963
d98a3884 964(define_insn "*umin<mode>3_neon"
965 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
966 (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
967 (match_operand:VDQIW 2 "s_register_operand" "w")))]
968 "TARGET_NEON"
bcaec148 969 "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
970 [(set_attr "neon_type" "neon_int_5")]
971)
d98a3884 972
973(define_insn "*umax<mode>3_neon"
974 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
975 (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
976 (match_operand:VDQIW 2 "s_register_operand" "w")))]
977 "TARGET_NEON"
bcaec148 978 "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
979 [(set_attr "neon_type" "neon_int_5")]
980)
d98a3884 981
982(define_insn "*smin<mode>3_neon"
983 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
984 (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
985 (match_operand:VDQW 2 "s_register_operand" "w")))]
986 "TARGET_NEON"
bcaec148 987 "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
988 [(set (attr "neon_type")
0bf497f5 989 (if_then_else (match_test "<Is_float_mode>")
bcaec148 990 (const_string "neon_fp_vadd_ddd_vabs_dd")
991 (const_string "neon_int_5")))]
992)
d98a3884 993
994(define_insn "*smax<mode>3_neon"
995 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
996 (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
997 (match_operand:VDQW 2 "s_register_operand" "w")))]
998 "TARGET_NEON"
bcaec148 999 "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1000 [(set (attr "neon_type")
0bf497f5 1001 (if_then_else (match_test "<Is_float_mode>")
bcaec148 1002 (const_string "neon_fp_vadd_ddd_vabs_dd")
1003 (const_string "neon_int_5")))]
1004)
d98a3884 1005
1006; TODO: V2DI shifts are current disabled because there are bugs in the
1007; generic vectorizer code. It ends up creating a V2DI constructor with
1008; SImode elements.
1009
6fbc081b 1010(define_insn "vashl<mode>3"
73f20323 1011 [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
1012 (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
1013 (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
1014 "TARGET_NEON"
1015 {
1016 switch (which_alternative)
1017 {
1018 case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
1019 case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
1020 <MODE>mode,
1021 VALID_NEON_QREG_MODE (<MODE>mode),
1022 true);
1023 default: gcc_unreachable ();
1024 }
1025 }
1026 [(set (attr "neon_type")
0bf497f5 1027 (if_then_else (match_test "<Is_d_reg>")
73f20323 1028 (const_string "neon_vshl_ddd")
1029 (const_string "neon_shift_3")))]
1030)
1031
1032(define_insn "vashr<mode>3_imm"
d98a3884 1033 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
73f20323 1034 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1035 (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
d98a3884 1036 "TARGET_NEON"
73f20323 1037 {
1038 return neon_output_shift_immediate ("vshr", 's', &operands[2],
1039 <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1040 false);
1041 }
bcaec148 1042 [(set (attr "neon_type")
0bf497f5 1043 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1044 (const_string "neon_vshl_ddd")
1045 (const_string "neon_shift_3")))]
1046)
d98a3884 1047
73f20323 1048(define_insn "vlshr<mode>3_imm"
1049 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1050 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1051 (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1052 "TARGET_NEON"
1053 {
1054 return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1055 <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1056 false);
1057 }
1058 [(set (attr "neon_type")
0bf497f5 1059 (if_then_else (match_test "<Is_d_reg>")
73f20323 1060 (const_string "neon_vshl_ddd")
1061 (const_string "neon_shift_3")))]
1062)
1063
d98a3884 1064; Used for implementing logical shift-right, which is a left-shift by a negative
1065; amount, with signed operands. This is essentially the same as ashl<mode>3
1066; above, but using an unspec in case GCC tries anything tricky with negative
1067; shift amounts.
1068
1069(define_insn "ashl<mode>3_signed"
1070 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1071 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1072 (match_operand:VDQI 2 "s_register_operand" "w")]
1073 UNSPEC_ASHIFT_SIGNED))]
1074 "TARGET_NEON"
bcaec148 1075 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1076 [(set (attr "neon_type")
0bf497f5 1077 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1078 (const_string "neon_vshl_ddd")
1079 (const_string "neon_shift_3")))]
1080)
d98a3884 1081
1082; Used for implementing logical shift-right, which is a left-shift by a negative
1083; amount, with unsigned operands.
1084
1085(define_insn "ashl<mode>3_unsigned"
1086 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1087 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1088 (match_operand:VDQI 2 "s_register_operand" "w")]
1089 UNSPEC_ASHIFT_UNSIGNED))]
1090 "TARGET_NEON"
bcaec148 1091 "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1092 [(set (attr "neon_type")
0bf497f5 1093 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1094 (const_string "neon_vshl_ddd")
1095 (const_string "neon_shift_3")))]
1096)
d98a3884 1097
6fbc081b 1098(define_expand "vashr<mode>3"
d98a3884 1099 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1100 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
73f20323 1101 (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
d98a3884 1102 "TARGET_NEON"
1103{
7510cf26 1104 if (s_register_operand (operands[2], <MODE>mode))
73f20323 1105 {
7510cf26 1106 rtx neg = gen_reg_rtx (<MODE>mode);
73f20323 1107 emit_insn (gen_neg<mode>2 (neg, operands[2]));
1108 emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1109 }
1110 else
1111 emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
d98a3884 1112 DONE;
1113})
1114
6fbc081b 1115(define_expand "vlshr<mode>3"
d98a3884 1116 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1117 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
73f20323 1118 (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
d98a3884 1119 "TARGET_NEON"
1120{
7510cf26 1121 if (s_register_operand (operands[2], <MODE>mode))
73f20323 1122 {
7510cf26 1123 rtx neg = gen_reg_rtx (<MODE>mode);
73f20323 1124 emit_insn (gen_neg<mode>2 (neg, operands[2]));
1125 emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1126 }
1127 else
1128 emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
d98a3884 1129 DONE;
1130})
1131
1132;; Widening operations
1133
1134(define_insn "widen_ssum<mode>3"
1135 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1136 (plus:<V_widen> (sign_extend:<V_widen>
1137 (match_operand:VW 1 "s_register_operand" "%w"))
1138 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1139 "TARGET_NEON"
bcaec148 1140 "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1141 [(set_attr "neon_type" "neon_int_3")]
1142)
d98a3884 1143
1144(define_insn "widen_usum<mode>3"
1145 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1146 (plus:<V_widen> (zero_extend:<V_widen>
1147 (match_operand:VW 1 "s_register_operand" "%w"))
1148 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1149 "TARGET_NEON"
bcaec148 1150 "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1151 [(set_attr "neon_type" "neon_int_3")]
1152)
d98a3884 1153
1154;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1155;; shift-count granularity. That's good enough for the middle-end's current
1156;; needs.
1157
b46a36c7 1158;; Note that it's not safe to perform such an operation in big-endian mode,
1159;; due to element-ordering issues.
1160
d98a3884 1161(define_expand "vec_shr_<mode>"
1162 [(match_operand:VDQ 0 "s_register_operand" "")
1163 (match_operand:VDQ 1 "s_register_operand" "")
1164 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
b46a36c7 1165 "TARGET_NEON && !BYTES_BIG_ENDIAN"
d98a3884 1166{
1167 rtx zero_reg;
1168 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1169 const int width = GET_MODE_BITSIZE (<MODE>mode);
1170 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1171 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1172 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1173
1174 if (num_bits == width)
1175 {
1176 emit_move_insn (operands[0], operands[1]);
1177 DONE;
1178 }
1179
1180 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1181 operands[0] = gen_lowpart (bvecmode, operands[0]);
1182 operands[1] = gen_lowpart (bvecmode, operands[1]);
1183
1184 emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1185 GEN_INT (num_bits / BITS_PER_UNIT)));
1186 DONE;
1187})
1188
1189(define_expand "vec_shl_<mode>"
1190 [(match_operand:VDQ 0 "s_register_operand" "")
1191 (match_operand:VDQ 1 "s_register_operand" "")
1192 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
b46a36c7 1193 "TARGET_NEON && !BYTES_BIG_ENDIAN"
d98a3884 1194{
1195 rtx zero_reg;
1196 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1197 const int width = GET_MODE_BITSIZE (<MODE>mode);
1198 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1199 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1200 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1201
1202 if (num_bits == 0)
1203 {
1204 emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1205 DONE;
1206 }
1207
1208 num_bits = width - num_bits;
1209
1210 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1211 operands[0] = gen_lowpart (bvecmode, operands[0]);
1212 operands[1] = gen_lowpart (bvecmode, operands[1]);
1213
1214 emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1215 GEN_INT (num_bits / BITS_PER_UNIT)));
1216 DONE;
1217})
1218
1219;; Helpers for quad-word reduction operations
1220
1221; Add (or smin, smax...) the low N/2 elements of the N-element vector
1222; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1223; N/2-element vector.
1224
1225(define_insn "quad_halves_<code>v4si"
1226 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1227 (vqh_ops:V2SI
1228 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1229 (parallel [(const_int 0) (const_int 1)]))
1230 (vec_select:V2SI (match_dup 1)
1231 (parallel [(const_int 2) (const_int 3)]))))]
1232 "TARGET_NEON"
bcaec148 1233 "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1234 [(set_attr "vqh_mnem" "<VQH_mnem>")
1235 (set (attr "neon_type")
1236 (if_then_else (eq_attr "vqh_mnem" "vadd")
1237 (const_string "neon_int_1") (const_string "neon_int_5")))]
1238)
d98a3884 1239
1240(define_insn "quad_halves_<code>v4sf"
1241 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1242 (vqhs_ops:V2SF
1243 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1244 (parallel [(const_int 0) (const_int 1)]))
1245 (vec_select:V2SF (match_dup 1)
1246 (parallel [(const_int 2) (const_int 3)]))))]
fe4048be 1247 "TARGET_NEON && flag_unsafe_math_optimizations"
bcaec148 1248 "<VQH_mnem>.f32\t%P0, %e1, %f1"
1249 [(set_attr "vqh_mnem" "<VQH_mnem>")
1250 (set (attr "neon_type")
1251 (if_then_else (eq_attr "vqh_mnem" "vadd")
1252 (const_string "neon_int_1") (const_string "neon_int_5")))]
1253)
d98a3884 1254
1255(define_insn "quad_halves_<code>v8hi"
1256 [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1257 (vqh_ops:V4HI
1258 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1259 (parallel [(const_int 0) (const_int 1)
1260 (const_int 2) (const_int 3)]))
1261 (vec_select:V4HI (match_dup 1)
1262 (parallel [(const_int 4) (const_int 5)
1263 (const_int 6) (const_int 7)]))))]
1264 "TARGET_NEON"
bcaec148 1265 "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1266 [(set_attr "vqh_mnem" "<VQH_mnem>")
1267 (set (attr "neon_type")
1268 (if_then_else (eq_attr "vqh_mnem" "vadd")
1269 (const_string "neon_int_1") (const_string "neon_int_5")))]
1270)
d98a3884 1271
1272(define_insn "quad_halves_<code>v16qi"
1273 [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1274 (vqh_ops:V8QI
1275 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1276 (parallel [(const_int 0) (const_int 1)
1277 (const_int 2) (const_int 3)
1278 (const_int 4) (const_int 5)
1279 (const_int 6) (const_int 7)]))
1280 (vec_select:V8QI (match_dup 1)
1281 (parallel [(const_int 8) (const_int 9)
1282 (const_int 10) (const_int 11)
1283 (const_int 12) (const_int 13)
1284 (const_int 14) (const_int 15)]))))]
1285 "TARGET_NEON"
bcaec148 1286 "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1287 [(set_attr "vqh_mnem" "<VQH_mnem>")
1288 (set (attr "neon_type")
1289 (if_then_else (eq_attr "vqh_mnem" "vadd")
1290 (const_string "neon_int_1") (const_string "neon_int_5")))]
1291)
d98a3884 1292
a62cc977 1293(define_expand "move_hi_quad_<mode>"
1294 [(match_operand:ANY128 0 "s_register_operand" "")
1295 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1296 "TARGET_NEON"
d98a3884 1297{
be7a395b 1298 emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1299 GET_MODE_SIZE (<V_HALF>mode)),
1300 operands[1]);
a62cc977 1301 DONE;
1302})
1303
1304(define_expand "move_lo_quad_<mode>"
1305 [(match_operand:ANY128 0 "s_register_operand" "")
1306 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1307 "TARGET_NEON"
d98a3884 1308{
be7a395b 1309 emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1310 <MODE>mode, 0),
1311 operands[1]);
a62cc977 1312 DONE;
1313})
d98a3884 1314
1315;; Reduction operations
1316
1317(define_expand "reduc_splus_<mode>"
1318 [(match_operand:VD 0 "s_register_operand" "")
1319 (match_operand:VD 1 "s_register_operand" "")]
fe4048be 1320 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
d98a3884 1321{
1322 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1323 &gen_neon_vpadd_internal<mode>);
1324 DONE;
1325})
1326
1327(define_expand "reduc_splus_<mode>"
1328 [(match_operand:VQ 0 "s_register_operand" "")
1329 (match_operand:VQ 1 "s_register_operand" "")]
b46a36c7 1330 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1331 && !BYTES_BIG_ENDIAN"
d98a3884 1332{
1333 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1334 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1335
1336 emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1337 emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1338 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1339
1340 DONE;
1341})
1342
1343(define_insn "reduc_splus_v2di"
1344 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1345 (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1346 UNSPEC_VPADD))]
b46a36c7 1347 "TARGET_NEON && !BYTES_BIG_ENDIAN"
bcaec148 1348 "vadd.i64\t%e0, %e1, %f1"
1349 [(set_attr "neon_type" "neon_int_1")]
1350)
d98a3884 1351
1352;; NEON does not distinguish between signed and unsigned addition except on
1353;; widening operations.
1354(define_expand "reduc_uplus_<mode>"
1355 [(match_operand:VDQI 0 "s_register_operand" "")
1356 (match_operand:VDQI 1 "s_register_operand" "")]
b46a36c7 1357 "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
d98a3884 1358{
1359 emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1360 DONE;
1361})
1362
1363(define_expand "reduc_smin_<mode>"
1364 [(match_operand:VD 0 "s_register_operand" "")
1365 (match_operand:VD 1 "s_register_operand" "")]
fe4048be 1366 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
d98a3884 1367{
1368 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1369 &gen_neon_vpsmin<mode>);
1370 DONE;
1371})
1372
1373(define_expand "reduc_smin_<mode>"
1374 [(match_operand:VQ 0 "s_register_operand" "")
1375 (match_operand:VQ 1 "s_register_operand" "")]
b46a36c7 1376 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1377 && !BYTES_BIG_ENDIAN"
d98a3884 1378{
1379 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1380 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1381
1382 emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1383 emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1384 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1385
1386 DONE;
1387})
1388
1389(define_expand "reduc_smax_<mode>"
1390 [(match_operand:VD 0 "s_register_operand" "")
1391 (match_operand:VD 1 "s_register_operand" "")]
fe4048be 1392 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
d98a3884 1393{
1394 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1395 &gen_neon_vpsmax<mode>);
1396 DONE;
1397})
1398
1399(define_expand "reduc_smax_<mode>"
1400 [(match_operand:VQ 0 "s_register_operand" "")
1401 (match_operand:VQ 1 "s_register_operand" "")]
b46a36c7 1402 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1403 && !BYTES_BIG_ENDIAN"
d98a3884 1404{
1405 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1406 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1407
1408 emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1409 emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1410 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1411
1412 DONE;
1413})
1414
1415(define_expand "reduc_umin_<mode>"
1416 [(match_operand:VDI 0 "s_register_operand" "")
1417 (match_operand:VDI 1 "s_register_operand" "")]
1418 "TARGET_NEON"
1419{
1420 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1421 &gen_neon_vpumin<mode>);
1422 DONE;
1423})
1424
1425(define_expand "reduc_umin_<mode>"
1426 [(match_operand:VQI 0 "s_register_operand" "")
1427 (match_operand:VQI 1 "s_register_operand" "")]
b46a36c7 1428 "TARGET_NEON && !BYTES_BIG_ENDIAN"
d98a3884 1429{
1430 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1431 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1432
1433 emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1434 emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1435 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1436
1437 DONE;
1438})
1439
1440(define_expand "reduc_umax_<mode>"
1441 [(match_operand:VDI 0 "s_register_operand" "")
1442 (match_operand:VDI 1 "s_register_operand" "")]
1443 "TARGET_NEON"
1444{
1445 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1446 &gen_neon_vpumax<mode>);
1447 DONE;
1448})
1449
1450(define_expand "reduc_umax_<mode>"
1451 [(match_operand:VQI 0 "s_register_operand" "")
1452 (match_operand:VQI 1 "s_register_operand" "")]
b46a36c7 1453 "TARGET_NEON && !BYTES_BIG_ENDIAN"
d98a3884 1454{
1455 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1456 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1457
1458 emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1459 emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1460 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1461
1462 DONE;
1463})
1464
1465(define_insn "neon_vpadd_internal<mode>"
1466 [(set (match_operand:VD 0 "s_register_operand" "=w")
1467 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1468 (match_operand:VD 2 "s_register_operand" "w")]
1469 UNSPEC_VPADD))]
1470 "TARGET_NEON"
bcaec148 1471 "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1472 ;; Assume this schedules like vadd.
1473 [(set (attr "neon_type")
0bf497f5 1474 (if_then_else (match_test "<Is_float_mode>")
1475 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1476 (const_string "neon_fp_vadd_ddd_vabs_dd")
1477 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1478 (const_string "neon_int_1")))]
1479)
d98a3884 1480
1481(define_insn "neon_vpsmin<mode>"
1482 [(set (match_operand:VD 0 "s_register_operand" "=w")
1483 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1484 (match_operand:VD 2 "s_register_operand" "w")]
1485 UNSPEC_VPSMIN))]
1486 "TARGET_NEON"
bcaec148 1487 "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1488 ;; Assume this schedules like vmin.
1489 [(set (attr "neon_type")
0bf497f5 1490 (if_then_else (match_test "<Is_float_mode>")
bcaec148 1491 (const_string "neon_fp_vadd_ddd_vabs_dd")
1492 (const_string "neon_int_5")))]
1493)
d98a3884 1494
1495(define_insn "neon_vpsmax<mode>"
1496 [(set (match_operand:VD 0 "s_register_operand" "=w")
1497 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1498 (match_operand:VD 2 "s_register_operand" "w")]
1499 UNSPEC_VPSMAX))]
1500 "TARGET_NEON"
bcaec148 1501 "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1502 ;; Assume this schedules like vmax.
1503 [(set (attr "neon_type")
0bf497f5 1504 (if_then_else (match_test "<Is_float_mode>")
bcaec148 1505 (const_string "neon_fp_vadd_ddd_vabs_dd")
1506 (const_string "neon_int_5")))]
1507)
d98a3884 1508
1509(define_insn "neon_vpumin<mode>"
1510 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1511 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1512 (match_operand:VDI 2 "s_register_operand" "w")]
1513 UNSPEC_VPUMIN))]
1514 "TARGET_NEON"
bcaec148 1515 "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1516 ;; Assume this schedules like umin.
1517 [(set_attr "neon_type" "neon_int_5")]
1518)
d98a3884 1519
1520(define_insn "neon_vpumax<mode>"
1521 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1522 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1523 (match_operand:VDI 2 "s_register_operand" "w")]
1524 UNSPEC_VPUMAX))]
1525 "TARGET_NEON"
bcaec148 1526 "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1527 ;; Assume this schedules like umax.
1528 [(set_attr "neon_type" "neon_int_5")]
1529)
d98a3884 1530
1531;; Saturating arithmetic
1532
1533; NOTE: Neon supports many more saturating variants of instructions than the
1534; following, but these are all GCC currently understands.
1535; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1536; yet either, although these patterns may be used by intrinsics when they're
1537; added.
1538
1539(define_insn "*ss_add<mode>_neon"
1540 [(set (match_operand:VD 0 "s_register_operand" "=w")
1541 (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1542 (match_operand:VD 2 "s_register_operand" "w")))]
1543 "TARGET_NEON"
bcaec148 1544 "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1545 [(set_attr "neon_type" "neon_int_4")]
1546)
d98a3884 1547
1548(define_insn "*us_add<mode>_neon"
1549 [(set (match_operand:VD 0 "s_register_operand" "=w")
1550 (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1551 (match_operand:VD 2 "s_register_operand" "w")))]
1552 "TARGET_NEON"
bcaec148 1553 "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1554 [(set_attr "neon_type" "neon_int_4")]
1555)
d98a3884 1556
1557(define_insn "*ss_sub<mode>_neon"
1558 [(set (match_operand:VD 0 "s_register_operand" "=w")
1559 (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1560 (match_operand:VD 2 "s_register_operand" "w")))]
1561 "TARGET_NEON"
bcaec148 1562 "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1563 [(set_attr "neon_type" "neon_int_5")]
1564)
d98a3884 1565
1566(define_insn "*us_sub<mode>_neon"
1567 [(set (match_operand:VD 0 "s_register_operand" "=w")
1568 (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1569 (match_operand:VD 2 "s_register_operand" "w")))]
1570 "TARGET_NEON"
bcaec148 1571 "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1572 [(set_attr "neon_type" "neon_int_5")]
1573)
d98a3884 1574
0a987353 1575;; Conditional instructions. These are comparisons with conditional moves for
1576;; vectors. They perform the assignment:
1577;;
1578;; Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1579;;
1580;; where op3 is <, <=, ==, !=, >= or >. Operations are performed
1581;; element-wise.
1582
d6b19f6b 1583(define_expand "vcond<mode><mode>"
0a987353 1584 [(set (match_operand:VDQW 0 "s_register_operand" "")
1585 (if_then_else:VDQW
1586 (match_operator 3 "arm_comparison_operator"
1587 [(match_operand:VDQW 4 "s_register_operand" "")
1588 (match_operand:VDQW 5 "nonmemory_operand" "")])
1589 (match_operand:VDQW 1 "s_register_operand" "")
1590 (match_operand:VDQW 2 "s_register_operand" "")))]
1591 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1592{
1593 rtx mask;
1594 int inverse = 0, immediate_zero = 0;
1595 /* See the description of "magic" bits in the 'T' case of
1596 arm_print_operand. */
1597 HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1598 ? 3 : 1;
1599 rtx magic_rtx = GEN_INT (magic_word);
1600
1601 mask = gen_reg_rtx (<V_cmp_result>mode);
1602
1603 if (operands[5] == CONST0_RTX (<MODE>mode))
1604 immediate_zero = 1;
1605 else if (!REG_P (operands[5]))
1606 operands[5] = force_reg (<MODE>mode, operands[5]);
1607
1608 switch (GET_CODE (operands[3]))
1609 {
1610 case GE:
1611 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1612 magic_rtx));
1613 break;
1614
1615 case GT:
1616 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1617 magic_rtx));
1618 break;
1619
1620 case EQ:
1621 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1622 magic_rtx));
1623 break;
1624
1625 case LE:
1626 if (immediate_zero)
1627 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1628 magic_rtx));
1629 else
1630 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1631 magic_rtx));
1632 break;
1633
1634 case LT:
1635 if (immediate_zero)
1636 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1637 magic_rtx));
1638 else
1639 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1640 magic_rtx));
1641 break;
1642
1643 case NE:
1644 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1645 magic_rtx));
1646 inverse = 1;
1647 break;
1648
1649 default:
1650 gcc_unreachable ();
1651 }
1652
1653 if (inverse)
1654 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1655 operands[1]));
1656 else
1657 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1658 operands[2]));
1659
1660 DONE;
1661})
1662
d6b19f6b 1663(define_expand "vcondu<mode><mode>"
0a987353 1664 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1665 (if_then_else:VDQIW
1666 (match_operator 3 "arm_comparison_operator"
1667 [(match_operand:VDQIW 4 "s_register_operand" "")
1668 (match_operand:VDQIW 5 "s_register_operand" "")])
1669 (match_operand:VDQIW 1 "s_register_operand" "")
1670 (match_operand:VDQIW 2 "s_register_operand" "")))]
1671 "TARGET_NEON"
1672{
1673 rtx mask;
1674 int inverse = 0, immediate_zero = 0;
1675
1676 mask = gen_reg_rtx (<V_cmp_result>mode);
1677
1678 if (operands[5] == CONST0_RTX (<MODE>mode))
1679 immediate_zero = 1;
1680 else if (!REG_P (operands[5]))
1681 operands[5] = force_reg (<MODE>mode, operands[5]);
1682
1683 switch (GET_CODE (operands[3]))
1684 {
1685 case GEU:
1686 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1687 const0_rtx));
1688 break;
1689
1690 case GTU:
1691 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1692 const0_rtx));
1693 break;
1694
1695 case EQ:
1696 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1697 const0_rtx));
1698 break;
1699
1700 case LEU:
1701 if (immediate_zero)
1702 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1703 const0_rtx));
1704 else
1705 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1706 const0_rtx));
1707 break;
1708
1709 case LTU:
1710 if (immediate_zero)
1711 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1712 const0_rtx));
1713 else
1714 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1715 const0_rtx));
1716 break;
1717
1718 case NE:
1719 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1720 const0_rtx));
1721 inverse = 1;
1722 break;
1723
1724 default:
1725 gcc_unreachable ();
1726 }
1727
1728 if (inverse)
1729 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1730 operands[1]));
1731 else
1732 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1733 operands[2]));
1734
1735 DONE;
1736})
1737
d98a3884 1738;; Patterns for builtins.
1739
1740; good for plain vadd, vaddq.
1741
94829feb 1742(define_expand "neon_vadd<mode>"
1743 [(match_operand:VDQX 0 "s_register_operand" "=w")
1744 (match_operand:VDQX 1 "s_register_operand" "w")
1745 (match_operand:VDQX 2 "s_register_operand" "w")
1746 (match_operand:SI 3 "immediate_operand" "i")]
1747 "TARGET_NEON"
1748{
1749 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1750 emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1751 else
1752 emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1753 operands[2]));
1754 DONE;
1755})
1756
1757; Note that NEON operations don't support the full IEEE 754 standard: in
1758; particular, denormal values are flushed to zero. This means that GCC cannot
1759; use those instructions for autovectorization, etc. unless
1760; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1761; behaviour is permissible). Intrinsic operations (provided by the arm_neon.h
1762; header) must work in either case: if -funsafe-math-optimizations is given,
1763; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1764; expand to unspecs (which may potentially limit the extent to which they might
1765; be optimized by generic code).
1766
1767; Used for intrinsics when flag_unsafe_math_optimizations is false.
1768
1769(define_insn "neon_vadd<mode>_unspec"
d98a3884 1770 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1771 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
94829feb 1772 (match_operand:VDQX 2 "s_register_operand" "w")]
d98a3884 1773 UNSPEC_VADD))]
1774 "TARGET_NEON"
bcaec148 1775 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1776 [(set (attr "neon_type")
0bf497f5 1777 (if_then_else (match_test "<Is_float_mode>")
1778 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1779 (const_string "neon_fp_vadd_ddd_vabs_dd")
1780 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1781 (const_string "neon_int_1")))]
1782)
d98a3884 1783
1784; operand 3 represents in bits:
1785; bit 0: signed (vs unsigned).
1786; bit 1: rounding (vs none).
1787
1788(define_insn "neon_vaddl<mode>"
1789 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1790 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1791 (match_operand:VDI 2 "s_register_operand" "w")
1792 (match_operand:SI 3 "immediate_operand" "i")]
1793 UNSPEC_VADDL))]
1794 "TARGET_NEON"
bcaec148 1795 "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1796 [(set_attr "neon_type" "neon_int_3")]
1797)
d98a3884 1798
1799(define_insn "neon_vaddw<mode>"
1800 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1801 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1802 (match_operand:VDI 2 "s_register_operand" "w")
1803 (match_operand:SI 3 "immediate_operand" "i")]
1804 UNSPEC_VADDW))]
1805 "TARGET_NEON"
bcaec148 1806 "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1807 [(set_attr "neon_type" "neon_int_2")]
1808)
d98a3884 1809
1810; vhadd and vrhadd.
1811
1812(define_insn "neon_vhadd<mode>"
1813 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1814 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1815 (match_operand:VDQIW 2 "s_register_operand" "w")
1816 (match_operand:SI 3 "immediate_operand" "i")]
1817 UNSPEC_VHADD))]
1818 "TARGET_NEON"
bcaec148 1819 "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1820 [(set_attr "neon_type" "neon_int_4")]
1821)
d98a3884 1822
1823(define_insn "neon_vqadd<mode>"
1824 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1825 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1826 (match_operand:VDQIX 2 "s_register_operand" "w")
1827 (match_operand:SI 3 "immediate_operand" "i")]
1828 UNSPEC_VQADD))]
1829 "TARGET_NEON"
bcaec148 1830 "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1831 [(set_attr "neon_type" "neon_int_4")]
1832)
d98a3884 1833
1834(define_insn "neon_vaddhn<mode>"
1835 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1836 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1837 (match_operand:VN 2 "s_register_operand" "w")
1838 (match_operand:SI 3 "immediate_operand" "i")]
1839 UNSPEC_VADDHN))]
1840 "TARGET_NEON"
bcaec148 1841 "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1842 [(set_attr "neon_type" "neon_int_4")]
1843)
d98a3884 1844
94829feb 1845;; We cannot replace this unspec with mul<mode>3 because of the odd
1846;; polynomial multiplication case that can specified by operand 3.
d98a3884 1847(define_insn "neon_vmul<mode>"
1848 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1849 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1850 (match_operand:VDQW 2 "s_register_operand" "w")
1851 (match_operand:SI 3 "immediate_operand" "i")]
1852 UNSPEC_VMUL))]
1853 "TARGET_NEON"
bcaec148 1854 "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1855 [(set (attr "neon_type")
0bf497f5 1856 (if_then_else (match_test "<Is_float_mode>")
1857 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1858 (const_string "neon_fp_vadd_ddd_vabs_dd")
1859 (const_string "neon_fp_vadd_qqq_vabs_qq"))
0bf497f5 1860 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1861 (if_then_else
0bf497f5 1862 (match_test "<Scalar_mul_8_16>")
bcaec148 1863 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1864 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
0bf497f5 1865 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 1866 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1867 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1868)
d98a3884 1869
94829feb 1870(define_expand "neon_vmla<mode>"
1871 [(match_operand:VDQW 0 "s_register_operand" "=w")
1872 (match_operand:VDQW 1 "s_register_operand" "0")
1873 (match_operand:VDQW 2 "s_register_operand" "w")
1874 (match_operand:VDQW 3 "s_register_operand" "w")
1875 (match_operand:SI 4 "immediate_operand" "i")]
1876 "TARGET_NEON"
1877{
1878 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1879 emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1880 operands[2], operands[3]));
1881 else
1882 emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1883 operands[2], operands[3]));
1884 DONE;
1885})
1886
1887; Used for intrinsics when flag_unsafe_math_optimizations is false.
1888
1889(define_insn "neon_vmla<mode>_unspec"
1890 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1891 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1892 (match_operand:VDQ 2 "s_register_operand" "w")
1893 (match_operand:VDQ 3 "s_register_operand" "w")]
1894 UNSPEC_VMLA))]
d98a3884 1895 "TARGET_NEON"
bcaec148 1896 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1897 [(set (attr "neon_type")
0bf497f5 1898 (if_then_else (match_test "<Is_float_mode>")
1899 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1900 (const_string "neon_fp_vmla_ddd")
1901 (const_string "neon_fp_vmla_qqq"))
0bf497f5 1902 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1903 (if_then_else
0bf497f5 1904 (match_test "<Scalar_mul_8_16>")
bcaec148 1905 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1906 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
0bf497f5 1907 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 1908 (const_string "neon_mla_qqq_8_16")
1909 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1910)
d98a3884 1911
1912(define_insn "neon_vmlal<mode>"
1913 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1914 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1915 (match_operand:VW 2 "s_register_operand" "w")
1916 (match_operand:VW 3 "s_register_operand" "w")
1917 (match_operand:SI 4 "immediate_operand" "i")]
1918 UNSPEC_VMLAL))]
1919 "TARGET_NEON"
bcaec148 1920 "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1921 [(set (attr "neon_type")
0bf497f5 1922 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 1923 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1924 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1925)
d98a3884 1926
94829feb 1927(define_expand "neon_vmls<mode>"
1928 [(match_operand:VDQW 0 "s_register_operand" "=w")
1929 (match_operand:VDQW 1 "s_register_operand" "0")
1930 (match_operand:VDQW 2 "s_register_operand" "w")
1931 (match_operand:VDQW 3 "s_register_operand" "w")
1932 (match_operand:SI 4 "immediate_operand" "i")]
1933 "TARGET_NEON"
1934{
1935 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1936 emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1937 operands[1], operands[2], operands[3]));
1938 else
1939 emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1940 operands[2], operands[3]));
1941 DONE;
1942})
1943
1944; Used for intrinsics when flag_unsafe_math_optimizations is false.
1945
1946(define_insn "neon_vmls<mode>_unspec"
1947 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1948 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1949 (match_operand:VDQ 2 "s_register_operand" "w")
1950 (match_operand:VDQ 3 "s_register_operand" "w")]
1951 UNSPEC_VMLS))]
d98a3884 1952 "TARGET_NEON"
bcaec148 1953 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1954 [(set (attr "neon_type")
0bf497f5 1955 (if_then_else (match_test "<Is_float_mode>")
1956 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1957 (const_string "neon_fp_vmla_ddd")
1958 (const_string "neon_fp_vmla_qqq"))
0bf497f5 1959 (if_then_else (match_test "<Is_d_reg>")
bcaec148 1960 (if_then_else
0bf497f5 1961 (match_test "<Scalar_mul_8_16>")
bcaec148 1962 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1963 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1964 (if_then_else
0bf497f5 1965 (match_test "<Scalar_mul_8_16>")
bcaec148 1966 (const_string "neon_mla_qqq_8_16")
1967 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1968)
d98a3884 1969
1970(define_insn "neon_vmlsl<mode>"
1971 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1972 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1973 (match_operand:VW 2 "s_register_operand" "w")
1974 (match_operand:VW 3 "s_register_operand" "w")
1975 (match_operand:SI 4 "immediate_operand" "i")]
1976 UNSPEC_VMLSL))]
1977 "TARGET_NEON"
bcaec148 1978 "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1979 [(set (attr "neon_type")
0bf497f5 1980 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 1981 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1982 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1983)
d98a3884 1984
1985(define_insn "neon_vqdmulh<mode>"
1986 [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
1987 (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
1988 (match_operand:VMDQI 2 "s_register_operand" "w")
1989 (match_operand:SI 3 "immediate_operand" "i")]
1990 UNSPEC_VQDMULH))]
1991 "TARGET_NEON"
bcaec148 1992 "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1993 [(set (attr "neon_type")
0bf497f5 1994 (if_then_else (match_test "<Is_d_reg>")
1995 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 1996 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1997 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
0bf497f5 1998 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 1999 (const_string "neon_mul_qqq_8_16_32_ddd_32")
2000 (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2001)
d98a3884 2002
2003(define_insn "neon_vqdmlal<mode>"
2004 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2005 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2006 (match_operand:VMDI 2 "s_register_operand" "w")
2007 (match_operand:VMDI 3 "s_register_operand" "w")
2008 (match_operand:SI 4 "immediate_operand" "i")]
2009 UNSPEC_VQDMLAL))]
2010 "TARGET_NEON"
bcaec148 2011 "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2012 [(set (attr "neon_type")
0bf497f5 2013 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 2014 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2015 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2016)
d98a3884 2017
2018(define_insn "neon_vqdmlsl<mode>"
2019 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2020 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2021 (match_operand:VMDI 2 "s_register_operand" "w")
2022 (match_operand:VMDI 3 "s_register_operand" "w")
2023 (match_operand:SI 4 "immediate_operand" "i")]
2024 UNSPEC_VQDMLSL))]
2025 "TARGET_NEON"
bcaec148 2026 "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2027 [(set (attr "neon_type")
0bf497f5 2028 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 2029 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2030 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2031)
d98a3884 2032
2033(define_insn "neon_vmull<mode>"
2034 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2035 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2036 (match_operand:VW 2 "s_register_operand" "w")
2037 (match_operand:SI 3 "immediate_operand" "i")]
2038 UNSPEC_VMULL))]
2039 "TARGET_NEON"
bcaec148 2040 "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2041 [(set (attr "neon_type")
0bf497f5 2042 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 2043 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2044 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2045)
d98a3884 2046
2047(define_insn "neon_vqdmull<mode>"
2048 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2049 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2050 (match_operand:VMDI 2 "s_register_operand" "w")
2051 (match_operand:SI 3 "immediate_operand" "i")]
2052 UNSPEC_VQDMULL))]
2053 "TARGET_NEON"
bcaec148 2054 "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2055 [(set (attr "neon_type")
0bf497f5 2056 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 2057 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2058 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2059)
d98a3884 2060
94829feb 2061(define_expand "neon_vsub<mode>"
2062 [(match_operand:VDQX 0 "s_register_operand" "=w")
2063 (match_operand:VDQX 1 "s_register_operand" "w")
2064 (match_operand:VDQX 2 "s_register_operand" "w")
2065 (match_operand:SI 3 "immediate_operand" "i")]
2066 "TARGET_NEON"
2067{
2068 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2069 emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2070 else
2071 emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2072 operands[2]));
2073 DONE;
2074})
2075
2076; Used for intrinsics when flag_unsafe_math_optimizations is false.
2077
2078(define_insn "neon_vsub<mode>_unspec"
d98a3884 2079 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2080 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
94829feb 2081 (match_operand:VDQX 2 "s_register_operand" "w")]
d98a3884 2082 UNSPEC_VSUB))]
2083 "TARGET_NEON"
bcaec148 2084 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2085 [(set (attr "neon_type")
0bf497f5 2086 (if_then_else (match_test "<Is_float_mode>")
2087 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2088 (const_string "neon_fp_vadd_ddd_vabs_dd")
2089 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2090 (const_string "neon_int_2")))]
2091)
d98a3884 2092
2093(define_insn "neon_vsubl<mode>"
2094 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2095 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2096 (match_operand:VDI 2 "s_register_operand" "w")
2097 (match_operand:SI 3 "immediate_operand" "i")]
2098 UNSPEC_VSUBL))]
2099 "TARGET_NEON"
bcaec148 2100 "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2101 [(set_attr "neon_type" "neon_int_2")]
2102)
d98a3884 2103
2104(define_insn "neon_vsubw<mode>"
2105 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2106 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2107 (match_operand:VDI 2 "s_register_operand" "w")
2108 (match_operand:SI 3 "immediate_operand" "i")]
2109 UNSPEC_VSUBW))]
2110 "TARGET_NEON"
bcaec148 2111 "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2112 [(set_attr "neon_type" "neon_int_2")]
2113)
d98a3884 2114
2115(define_insn "neon_vqsub<mode>"
2116 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2117 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2118 (match_operand:VDQIX 2 "s_register_operand" "w")
2119 (match_operand:SI 3 "immediate_operand" "i")]
2120 UNSPEC_VQSUB))]
2121 "TARGET_NEON"
bcaec148 2122 "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2123 [(set_attr "neon_type" "neon_int_5")]
2124)
d98a3884 2125
2126(define_insn "neon_vhsub<mode>"
2127 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2128 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2129 (match_operand:VDQIW 2 "s_register_operand" "w")
2130 (match_operand:SI 3 "immediate_operand" "i")]
2131 UNSPEC_VHSUB))]
2132 "TARGET_NEON"
bcaec148 2133 "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2134 [(set_attr "neon_type" "neon_int_5")]
2135)
d98a3884 2136
2137(define_insn "neon_vsubhn<mode>"
2138 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2139 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2140 (match_operand:VN 2 "s_register_operand" "w")
2141 (match_operand:SI 3 "immediate_operand" "i")]
2142 UNSPEC_VSUBHN))]
2143 "TARGET_NEON"
bcaec148 2144 "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2145 [(set_attr "neon_type" "neon_int_4")]
2146)
d98a3884 2147
2148(define_insn "neon_vceq<mode>"
0a987353 2149 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2150 (unspec:<V_cmp_result>
2151 [(match_operand:VDQW 1 "s_register_operand" "w,w")
1987d213 2152 (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
0a987353 2153 (match_operand:SI 3 "immediate_operand" "i,i")]
2154 UNSPEC_VCEQ))]
d98a3884 2155 "TARGET_NEON"
0a987353 2156 "@
2157 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2158 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
bcaec148 2159 [(set (attr "neon_type")
0bf497f5 2160 (if_then_else (match_test "<Is_float_mode>")
2161 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2162 (const_string "neon_fp_vadd_ddd_vabs_dd")
2163 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2164 (const_string "neon_int_5")))]
2165)
d98a3884 2166
2167(define_insn "neon_vcge<mode>"
0a987353 2168 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2169 (unspec:<V_cmp_result>
2170 [(match_operand:VDQW 1 "s_register_operand" "w,w")
1987d213 2171 (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
0a987353 2172 (match_operand:SI 3 "immediate_operand" "i,i")]
2173 UNSPEC_VCGE))]
d98a3884 2174 "TARGET_NEON"
0a987353 2175 "@
2176 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2177 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
bcaec148 2178 [(set (attr "neon_type")
0bf497f5 2179 (if_then_else (match_test "<Is_float_mode>")
2180 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2181 (const_string "neon_fp_vadd_ddd_vabs_dd")
2182 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2183 (const_string "neon_int_5")))]
2184)
d98a3884 2185
ca6c837f 2186(define_insn "neon_vcgeu<mode>"
2187 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2188 (unspec:<V_cmp_result>
2189 [(match_operand:VDQIW 1 "s_register_operand" "w")
2190 (match_operand:VDQIW 2 "s_register_operand" "w")
2191 (match_operand:SI 3 "immediate_operand" "i")]
2192 UNSPEC_VCGEU))]
2193 "TARGET_NEON"
2194 "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2195 [(set_attr "neon_type" "neon_int_5")]
2196)
2197
d98a3884 2198(define_insn "neon_vcgt<mode>"
0a987353 2199 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2200 (unspec:<V_cmp_result>
2201 [(match_operand:VDQW 1 "s_register_operand" "w,w")
1987d213 2202 (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
0a987353 2203 (match_operand:SI 3 "immediate_operand" "i,i")]
2204 UNSPEC_VCGT))]
d98a3884 2205 "TARGET_NEON"
0a987353 2206 "@
2207 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2208 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
bcaec148 2209 [(set (attr "neon_type")
0bf497f5 2210 (if_then_else (match_test "<Is_float_mode>")
2211 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2212 (const_string "neon_fp_vadd_ddd_vabs_dd")
2213 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2214 (const_string "neon_int_5")))]
2215)
d98a3884 2216
ca6c837f 2217(define_insn "neon_vcgtu<mode>"
2218 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2219 (unspec:<V_cmp_result>
2220 [(match_operand:VDQIW 1 "s_register_operand" "w")
2221 (match_operand:VDQIW 2 "s_register_operand" "w")
2222 (match_operand:SI 3 "immediate_operand" "i")]
2223 UNSPEC_VCGTU))]
2224 "TARGET_NEON"
2225 "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2226 [(set_attr "neon_type" "neon_int_5")]
2227)
2228
0a987353 2229;; VCLE and VCLT only support comparisons with immediate zero (register
2230;; variants are VCGE and VCGT with operands reversed).
2231
2232(define_insn "neon_vcle<mode>"
2233 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2234 (unspec:<V_cmp_result>
2235 [(match_operand:VDQW 1 "s_register_operand" "w")
1987d213 2236 (match_operand:VDQW 2 "zero_operand" "Dz")
0a987353 2237 (match_operand:SI 3 "immediate_operand" "i")]
2238 UNSPEC_VCLE))]
2239 "TARGET_NEON"
2240 "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2241 [(set (attr "neon_type")
0bf497f5 2242 (if_then_else (match_test "<Is_float_mode>")
2243 (if_then_else (match_test "<Is_d_reg>")
0a987353 2244 (const_string "neon_fp_vadd_ddd_vabs_dd")
2245 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2246 (const_string "neon_int_5")))]
2247)
2248
2249(define_insn "neon_vclt<mode>"
2250 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2251 (unspec:<V_cmp_result>
2252 [(match_operand:VDQW 1 "s_register_operand" "w")
1987d213 2253 (match_operand:VDQW 2 "zero_operand" "Dz")
0a987353 2254 (match_operand:SI 3 "immediate_operand" "i")]
2255 UNSPEC_VCLT))]
2256 "TARGET_NEON"
2257 "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2258 [(set (attr "neon_type")
0bf497f5 2259 (if_then_else (match_test "<Is_float_mode>")
2260 (if_then_else (match_test "<Is_d_reg>")
0a987353 2261 (const_string "neon_fp_vadd_ddd_vabs_dd")
2262 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2263 (const_string "neon_int_5")))]
2264)
2265
d98a3884 2266(define_insn "neon_vcage<mode>"
2267 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2268 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2269 (match_operand:VCVTF 2 "s_register_operand" "w")
2270 (match_operand:SI 3 "immediate_operand" "i")]
2271 UNSPEC_VCAGE))]
2272 "TARGET_NEON"
bcaec148 2273 "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2274 [(set (attr "neon_type")
0bf497f5 2275 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2276 (const_string "neon_fp_vadd_ddd_vabs_dd")
2277 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2278)
d98a3884 2279
2280(define_insn "neon_vcagt<mode>"
2281 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2282 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2283 (match_operand:VCVTF 2 "s_register_operand" "w")
2284 (match_operand:SI 3 "immediate_operand" "i")]
2285 UNSPEC_VCAGT))]
2286 "TARGET_NEON"
bcaec148 2287 "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2288 [(set (attr "neon_type")
0bf497f5 2289 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2290 (const_string "neon_fp_vadd_ddd_vabs_dd")
2291 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2292)
d98a3884 2293
2294(define_insn "neon_vtst<mode>"
2295 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2296 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2297 (match_operand:VDQIW 2 "s_register_operand" "w")
2298 (match_operand:SI 3 "immediate_operand" "i")]
2299 UNSPEC_VTST))]
2300 "TARGET_NEON"
bcaec148 2301 "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2302 [(set_attr "neon_type" "neon_int_4")]
2303)
d98a3884 2304
2305(define_insn "neon_vabd<mode>"
2306 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2307 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2308 (match_operand:VDQW 2 "s_register_operand" "w")
2309 (match_operand:SI 3 "immediate_operand" "i")]
2310 UNSPEC_VABD))]
2311 "TARGET_NEON"
bcaec148 2312 "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2313 [(set (attr "neon_type")
0bf497f5 2314 (if_then_else (match_test "<Is_float_mode>")
2315 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2316 (const_string "neon_fp_vadd_ddd_vabs_dd")
2317 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2318 (const_string "neon_int_5")))]
2319)
d98a3884 2320
2321(define_insn "neon_vabdl<mode>"
2322 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2323 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2324 (match_operand:VW 2 "s_register_operand" "w")
2325 (match_operand:SI 3 "immediate_operand" "i")]
2326 UNSPEC_VABDL))]
2327 "TARGET_NEON"
bcaec148 2328 "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2329 [(set_attr "neon_type" "neon_int_5")]
2330)
d98a3884 2331
2332(define_insn "neon_vaba<mode>"
2333 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
8c619ffb 2334 (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
94829feb 2335 (match_operand:VDQIW 3 "s_register_operand" "w")
2336 (match_operand:SI 4 "immediate_operand" "i")]
8c619ffb 2337 UNSPEC_VABD)
2338 (match_operand:VDQIW 1 "s_register_operand" "0")))]
d98a3884 2339 "TARGET_NEON"
bcaec148 2340 "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2341 [(set (attr "neon_type")
0bf497f5 2342 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2343 (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2344)
d98a3884 2345
2346(define_insn "neon_vabal<mode>"
2347 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
8c619ffb 2348 (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
94829feb 2349 (match_operand:VW 3 "s_register_operand" "w")
2350 (match_operand:SI 4 "immediate_operand" "i")]
8c619ffb 2351 UNSPEC_VABDL)
2352 (match_operand:<V_widen> 1 "s_register_operand" "0")))]
d98a3884 2353 "TARGET_NEON"
bcaec148 2354 "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2355 [(set_attr "neon_type" "neon_vaba")]
2356)
d98a3884 2357
2358(define_insn "neon_vmax<mode>"
2359 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2360 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2361 (match_operand:VDQW 2 "s_register_operand" "w")
2362 (match_operand:SI 3 "immediate_operand" "i")]
2363 UNSPEC_VMAX))]
2364 "TARGET_NEON"
bcaec148 2365 "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2366 [(set (attr "neon_type")
0bf497f5 2367 (if_then_else (match_test "<Is_float_mode>")
2368 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2369 (const_string "neon_fp_vadd_ddd_vabs_dd")
2370 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2371 (const_string "neon_int_5")))]
2372)
d98a3884 2373
2374(define_insn "neon_vmin<mode>"
2375 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2376 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2377 (match_operand:VDQW 2 "s_register_operand" "w")
2378 (match_operand:SI 3 "immediate_operand" "i")]
2379 UNSPEC_VMIN))]
2380 "TARGET_NEON"
bcaec148 2381 "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2382 [(set (attr "neon_type")
0bf497f5 2383 (if_then_else (match_test "<Is_float_mode>")
2384 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2385 (const_string "neon_fp_vadd_ddd_vabs_dd")
2386 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2387 (const_string "neon_int_5")))]
2388)
d98a3884 2389
2390(define_expand "neon_vpadd<mode>"
2391 [(match_operand:VD 0 "s_register_operand" "=w")
2392 (match_operand:VD 1 "s_register_operand" "w")
2393 (match_operand:VD 2 "s_register_operand" "w")
2394 (match_operand:SI 3 "immediate_operand" "i")]
2395 "TARGET_NEON"
2396{
2397 emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2398 operands[2]));
2399 DONE;
2400})
2401
2402(define_insn "neon_vpaddl<mode>"
2403 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2404 (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2405 (match_operand:SI 2 "immediate_operand" "i")]
2406 UNSPEC_VPADDL))]
2407 "TARGET_NEON"
bcaec148 2408 "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2409 ;; Assume this schedules like vaddl.
2410 [(set_attr "neon_type" "neon_int_3")]
2411)
d98a3884 2412
2413(define_insn "neon_vpadal<mode>"
2414 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2415 (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2416 (match_operand:VDQIW 2 "s_register_operand" "w")
2417 (match_operand:SI 3 "immediate_operand" "i")]
2418 UNSPEC_VPADAL))]
2419 "TARGET_NEON"
bcaec148 2420 "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2421 ;; Assume this schedules like vpadd.
2422 [(set_attr "neon_type" "neon_int_1")]
2423)
d98a3884 2424
2425(define_insn "neon_vpmax<mode>"
2426 [(set (match_operand:VD 0 "s_register_operand" "=w")
2427 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2428 (match_operand:VD 2 "s_register_operand" "w")
2429 (match_operand:SI 3 "immediate_operand" "i")]
2430 UNSPEC_VPMAX))]
2431 "TARGET_NEON"
bcaec148 2432 "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2433 ;; Assume this schedules like vmax.
2434 [(set (attr "neon_type")
0bf497f5 2435 (if_then_else (match_test "<Is_float_mode>")
bcaec148 2436 (const_string "neon_fp_vadd_ddd_vabs_dd")
2437 (const_string "neon_int_5")))]
2438)
d98a3884 2439
2440(define_insn "neon_vpmin<mode>"
2441 [(set (match_operand:VD 0 "s_register_operand" "=w")
2442 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2443 (match_operand:VD 2 "s_register_operand" "w")
2444 (match_operand:SI 3 "immediate_operand" "i")]
2445 UNSPEC_VPMIN))]
2446 "TARGET_NEON"
bcaec148 2447 "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2448 ;; Assume this schedules like vmin.
2449 [(set (attr "neon_type")
0bf497f5 2450 (if_then_else (match_test "<Is_float_mode>")
bcaec148 2451 (const_string "neon_fp_vadd_ddd_vabs_dd")
2452 (const_string "neon_int_5")))]
2453)
d98a3884 2454
2455(define_insn "neon_vrecps<mode>"
2456 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2457 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2458 (match_operand:VCVTF 2 "s_register_operand" "w")
2459 (match_operand:SI 3 "immediate_operand" "i")]
2460 UNSPEC_VRECPS))]
2461 "TARGET_NEON"
bcaec148 2462 "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2463 [(set (attr "neon_type")
0bf497f5 2464 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2465 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2466 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2467)
d98a3884 2468
2469(define_insn "neon_vrsqrts<mode>"
2470 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2471 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2472 (match_operand:VCVTF 2 "s_register_operand" "w")
2473 (match_operand:SI 3 "immediate_operand" "i")]
2474 UNSPEC_VRSQRTS))]
2475 "TARGET_NEON"
bcaec148 2476 "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2477 [(set (attr "neon_type")
0bf497f5 2478 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2479 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2480 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2481)
d98a3884 2482
94829feb 2483(define_expand "neon_vabs<mode>"
2484 [(match_operand:VDQW 0 "s_register_operand" "")
2485 (match_operand:VDQW 1 "s_register_operand" "")
2486 (match_operand:SI 2 "immediate_operand" "")]
d98a3884 2487 "TARGET_NEON"
94829feb 2488{
2489 emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2490 DONE;
2491})
d98a3884 2492
2493(define_insn "neon_vqabs<mode>"
2494 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2495 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2496 (match_operand:SI 2 "immediate_operand" "i")]
2497 UNSPEC_VQABS))]
2498 "TARGET_NEON"
bcaec148 2499 "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2500 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2501)
d98a3884 2502
2503(define_expand "neon_vneg<mode>"
2504 [(match_operand:VDQW 0 "s_register_operand" "")
2505 (match_operand:VDQW 1 "s_register_operand" "")
2506 (match_operand:SI 2 "immediate_operand" "")]
2507 "TARGET_NEON"
2508{
2509 emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2510 DONE;
2511})
2512
2513(define_insn "neon_vqneg<mode>"
2514 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2515 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2516 (match_operand:SI 2 "immediate_operand" "i")]
2517 UNSPEC_VQNEG))]
2518 "TARGET_NEON"
bcaec148 2519 "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2520 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2521)
d98a3884 2522
2523(define_insn "neon_vcls<mode>"
2524 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2525 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2526 (match_operand:SI 2 "immediate_operand" "i")]
2527 UNSPEC_VCLS))]
2528 "TARGET_NEON"
bcaec148 2529 "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2530 [(set_attr "neon_type" "neon_int_1")]
2531)
d98a3884 2532
e2669ea7 2533(define_insn "clz<mode>2"
d98a3884 2534 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
e2669ea7 2535 (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
d98a3884 2536 "TARGET_NEON"
bcaec148 2537 "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2538 [(set_attr "neon_type" "neon_int_1")]
2539)
d98a3884 2540
e2669ea7 2541(define_expand "neon_vclz<mode>"
2542 [(match_operand:VDQIW 0 "s_register_operand" "")
2543 (match_operand:VDQIW 1 "s_register_operand" "")
2544 (match_operand:SI 2 "immediate_operand" "")]
2545 "TARGET_NEON"
2546{
2547 emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2548 DONE;
2549})
2550
2551(define_insn "popcount<mode>2"
d98a3884 2552 [(set (match_operand:VE 0 "s_register_operand" "=w")
e2669ea7 2553 (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
d98a3884 2554 "TARGET_NEON"
bcaec148 2555 "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2556 [(set_attr "neon_type" "neon_int_1")]
2557)
d98a3884 2558
e2669ea7 2559(define_expand "neon_vcnt<mode>"
2560 [(match_operand:VE 0 "s_register_operand" "=w")
2561 (match_operand:VE 1 "s_register_operand" "w")
2562 (match_operand:SI 2 "immediate_operand" "i")]
2563 "TARGET_NEON"
2564{
2565 emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2566 DONE;
2567})
2568
d98a3884 2569(define_insn "neon_vrecpe<mode>"
2570 [(set (match_operand:V32 0 "s_register_operand" "=w")
2571 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2572 (match_operand:SI 2 "immediate_operand" "i")]
2573 UNSPEC_VRECPE))]
2574 "TARGET_NEON"
bcaec148 2575 "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2576 [(set (attr "neon_type")
0bf497f5 2577 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2578 (const_string "neon_fp_vadd_ddd_vabs_dd")
2579 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2580)
d98a3884 2581
2582(define_insn "neon_vrsqrte<mode>"
2583 [(set (match_operand:V32 0 "s_register_operand" "=w")
2584 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2585 (match_operand:SI 2 "immediate_operand" "i")]
2586 UNSPEC_VRSQRTE))]
2587 "TARGET_NEON"
bcaec148 2588 "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2589 [(set (attr "neon_type")
0bf497f5 2590 (if_then_else (match_test "<Is_d_reg>")
bcaec148 2591 (const_string "neon_fp_vadd_ddd_vabs_dd")
2592 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2593)
d98a3884 2594
2595(define_expand "neon_vmvn<mode>"
2596 [(match_operand:VDQIW 0 "s_register_operand" "")
2597 (match_operand:VDQIW 1 "s_register_operand" "")
2598 (match_operand:SI 2 "immediate_operand" "")]
2599 "TARGET_NEON"
2600{
2601 emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2602 DONE;
2603})
2604
4c0b79b4 2605(define_insn "neon_vget_lane<mode>_sext_internal"
2606 [(set (match_operand:SI 0 "s_register_operand" "=r")
2607 (sign_extend:SI
2608 (vec_select:<V_elem>
2609 (match_operand:VD 1 "s_register_operand" "w")
2610 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
d98a3884 2611 "TARGET_NEON"
cdf93281 2612{
2613 if (BYTES_BIG_ENDIAN)
2614 {
2615 int elt = INTVAL (operands[2]);
2616 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2617 operands[2] = GEN_INT (elt);
2618 }
17ae1a66 2619 return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
cdf93281 2620}
17ae1a66 2621 [(set_attr "neon_type" "neon_bp_simple")]
bcaec148 2622)
d98a3884 2623
4c0b79b4 2624(define_insn "neon_vget_lane<mode>_zext_internal"
2625 [(set (match_operand:SI 0 "s_register_operand" "=r")
2626 (zero_extend:SI
2627 (vec_select:<V_elem>
2628 (match_operand:VD 1 "s_register_operand" "w")
2629 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2630 "TARGET_NEON"
cdf93281 2631{
2632 if (BYTES_BIG_ENDIAN)
2633 {
2634 int elt = INTVAL (operands[2]);
2635 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2636 operands[2] = GEN_INT (elt);
2637 }
17ae1a66 2638 return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
cdf93281 2639}
17ae1a66 2640 [(set_attr "neon_type" "neon_bp_simple")]
4c0b79b4 2641)
d98a3884 2642
4c0b79b4 2643(define_insn "neon_vget_lane<mode>_sext_internal"
2644 [(set (match_operand:SI 0 "s_register_operand" "=r")
2645 (sign_extend:SI
2646 (vec_select:<V_elem>
2647 (match_operand:VQ 1 "s_register_operand" "w")
2648 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
d98a3884 2649 "TARGET_NEON"
8521669a 2650{
4c0b79b4 2651 rtx ops[3];
2652 int regno = REGNO (operands[1]);
2653 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2654 unsigned int elt = INTVAL (operands[2]);
cdf93281 2655 unsigned int elt_adj = elt % halfelts;
2656
2657 if (BYTES_BIG_ENDIAN)
2658 elt_adj = halfelts - 1 - elt_adj;
4c0b79b4 2659
2660 ops[0] = operands[0];
2661 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
cdf93281 2662 ops[2] = GEN_INT (elt_adj);
17ae1a66 2663 output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
4c0b79b4 2664
2665 return "";
8521669a 2666}
17ae1a66 2667 [(set_attr "neon_type" "neon_bp_simple")]
bcaec148 2668)
d98a3884 2669
4c0b79b4 2670(define_insn "neon_vget_lane<mode>_zext_internal"
2671 [(set (match_operand:SI 0 "s_register_operand" "=r")
2672 (zero_extend:SI
2673 (vec_select:<V_elem>
2674 (match_operand:VQ 1 "s_register_operand" "w")
2675 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
d98a3884 2676 "TARGET_NEON"
2677{
4c0b79b4 2678 rtx ops[3];
d98a3884 2679 int regno = REGNO (operands[1]);
2680 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2681 unsigned int elt = INTVAL (operands[2]);
cdf93281 2682 unsigned int elt_adj = elt % halfelts;
2683
2684 if (BYTES_BIG_ENDIAN)
2685 elt_adj = halfelts - 1 - elt_adj;
d98a3884 2686
2687 ops[0] = operands[0];
2688 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
cdf93281 2689 ops[2] = GEN_INT (elt_adj);
17ae1a66 2690 output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
d98a3884 2691
2692 return "";
2693}
17ae1a66 2694 [(set_attr "neon_type" "neon_bp_simple")]
4c0b79b4 2695)
2696
2697(define_expand "neon_vget_lane<mode>"
2698 [(match_operand:<V_ext> 0 "s_register_operand" "")
2699 (match_operand:VDQW 1 "s_register_operand" "")
2700 (match_operand:SI 2 "immediate_operand" "")
2701 (match_operand:SI 3 "immediate_operand" "")]
2702 "TARGET_NEON"
2703{
2704 HOST_WIDE_INT magic = INTVAL (operands[3]);
2705 rtx insn;
2706
2707 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2708
cdf93281 2709 if (BYTES_BIG_ENDIAN)
2710 {
2711 /* The intrinsics are defined in terms of a model where the
2712 element ordering in memory is vldm order, whereas the generic
2713 RTL is defined in terms of a model where the element ordering
2714 in memory is array order. Convert the lane number to conform
2715 to this model. */
2716 unsigned int elt = INTVAL (operands[2]);
2717 unsigned int reg_nelts
2718 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2719 elt ^= reg_nelts - 1;
2720 operands[2] = GEN_INT (elt);
2721 }
2722
4c0b79b4 2723 if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2724 insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2725 else
2726 {
2727 if ((magic & 1) != 0)
2728 insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2729 operands[2]);
2730 else
2731 insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2732 operands[2]);
2733 }
2734 emit_insn (insn);
2735 DONE;
2736})
2737
2738; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2739; elements.
2740
79a83503 2741(define_expand "neon_vget_lanedi"
2742 [(match_operand:DI 0 "s_register_operand" "=r")
2743 (match_operand:DI 1 "s_register_operand" "w")
2744 (match_operand:SI 2 "immediate_operand" "i")
2745 (match_operand:SI 3 "immediate_operand" "i")]
4c0b79b4 2746 "TARGET_NEON"
2747{
2748 neon_lane_bounds (operands[2], 0, 1);
79a83503 2749 emit_move_insn (operands[0], operands[1]);
2750 DONE;
2751})
d98a3884 2752
79a83503 2753(define_expand "neon_vget_lanev2di"
cd4e07db 2754 [(match_operand:DI 0 "s_register_operand" "")
2755 (match_operand:V2DI 1 "s_register_operand" "")
2756 (match_operand:SI 2 "immediate_operand" "")
2757 (match_operand:SI 3 "immediate_operand" "")]
d98a3884 2758 "TARGET_NEON"
2759{
cd4e07db 2760 switch (INTVAL (operands[2]))
2761 {
2762 case 0:
2763 emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2764 break;
2765 case 1:
2766 emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2767 break;
2768 default:
2769 neon_lane_bounds (operands[2], 0, 1);
2770 FAIL;
2771 }
79a83503 2772 DONE;
2773})
8521669a 2774
79a83503 2775(define_expand "neon_vset_lane<mode>"
2776 [(match_operand:VDQ 0 "s_register_operand" "=w")
2777 (match_operand:<V_elem> 1 "s_register_operand" "r")
2778 (match_operand:VDQ 2 "s_register_operand" "0")
2779 (match_operand:SI 3 "immediate_operand" "i")]
d98a3884 2780 "TARGET_NEON"
2781{
d98a3884 2782 unsigned int elt = INTVAL (operands[3]);
79a83503 2783 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
d98a3884 2784
79a83503 2785 if (BYTES_BIG_ENDIAN)
2786 {
2787 unsigned int reg_nelts
2788 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2789 elt ^= reg_nelts - 1;
2790 }
8521669a 2791
79a83503 2792 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2793 GEN_INT (1 << elt), operands[2]));
2794 DONE;
2795})
d98a3884 2796
79a83503 2797; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
d98a3884 2798
79a83503 2799(define_expand "neon_vset_lanedi"
2800 [(match_operand:DI 0 "s_register_operand" "=w")
2801 (match_operand:DI 1 "s_register_operand" "r")
2802 (match_operand:DI 2 "s_register_operand" "0")
2803 (match_operand:SI 3 "immediate_operand" "i")]
d98a3884 2804 "TARGET_NEON"
2805{
79a83503 2806 neon_lane_bounds (operands[3], 0, 1);
2807 emit_move_insn (operands[0], operands[1]);
2808 DONE;
2809})
d98a3884 2810
2811(define_expand "neon_vcreate<mode>"
2812 [(match_operand:VDX 0 "s_register_operand" "")
2813 (match_operand:DI 1 "general_operand" "")]
2814 "TARGET_NEON"
2815{
2816 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2817 emit_move_insn (operands[0], src);
2818 DONE;
2819})
2820
2821(define_insn "neon_vdup_n<mode>"
c747abbb 2822 [(set (match_operand:VX 0 "s_register_operand" "=w")
79a83503 2823 (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
d98a3884 2824 "TARGET_NEON"
17ae1a66 2825 "vdup.<V_sz_elem>\t%<V_reg>0, %1"
bcaec148 2826 ;; Assume this schedules like vmov.
17ae1a66 2827 [(set_attr "neon_type" "neon_bp_simple")]
bcaec148 2828)
d98a3884 2829
c747abbb 2830(define_insn "neon_vdup_n<mode>"
2831 [(set (match_operand:V32 0 "s_register_operand" "=w,w")
79a83503 2832 (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
c747abbb 2833 "TARGET_NEON"
2834 "@
17ae1a66 2835 vdup.<V_sz_elem>\t%<V_reg>0, %1
2836 vdup.<V_sz_elem>\t%<V_reg>0, %y1"
c747abbb 2837 ;; Assume this schedules like vmov.
17ae1a66 2838 [(set_attr "neon_type" "neon_bp_simple")]
c747abbb 2839)
2840
79a83503 2841(define_expand "neon_vdup_ndi"
2842 [(match_operand:DI 0 "s_register_operand" "=w")
2843 (match_operand:DI 1 "s_register_operand" "r")]
d98a3884 2844 "TARGET_NEON"
79a83503 2845{
2846 emit_move_insn (operands[0], operands[1]);
2847 DONE;
2848}
bcaec148 2849)
d98a3884 2850
2851(define_insn "neon_vdup_nv2di"
79a83503 2852 [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2853 (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
d98a3884 2854 "TARGET_NEON"
79a83503 2855 "@
17ae1a66 2856 vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
2857 vmov\t%e0, %P1\;vmov\t%f0, %P1"
2858 [(set_attr "length" "8")
bcaec148 2859 (set_attr "neon_type" "neon_bp_simple")]
2860)
d98a3884 2861
79a83503 2862(define_insn "neon_vdup_lane<mode>_internal"
2863 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2864 (vec_duplicate:VDQW
2865 (vec_select:<V_elem>
2866 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2867 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
d98a3884 2868 "TARGET_NEON"
8521669a 2869{
79a83503 2870 if (BYTES_BIG_ENDIAN)
2871 {
2872 int elt = INTVAL (operands[2]);
2873 elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2874 operands[2] = GEN_INT (elt);
2875 }
2876 if (<Is_d_reg>)
2877 return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2878 else
2879 return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
8521669a 2880}
bcaec148 2881 ;; Assume this schedules like vmov.
2882 [(set_attr "neon_type" "neon_bp_simple")]
2883)
d98a3884 2884
79a83503 2885(define_expand "neon_vdup_lane<mode>"
2886 [(match_operand:VDQW 0 "s_register_operand" "=w")
2887 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2888 (match_operand:SI 2 "immediate_operand" "i")]
d98a3884 2889 "TARGET_NEON"
8521669a 2890{
79a83503 2891 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2892 if (BYTES_BIG_ENDIAN)
2893 {
2894 unsigned int elt = INTVAL (operands[2]);
2895 unsigned int reg_nelts
2896 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2897 elt ^= reg_nelts - 1;
2898 operands[2] = GEN_INT (elt);
2899 }
2900 emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2901 operands[2]));
2902 DONE;
2903})
d98a3884 2904
2905; Scalar index is ignored, since only zero is valid here.
2906(define_expand "neon_vdup_lanedi"
79a83503 2907 [(match_operand:DI 0 "s_register_operand" "=w")
2908 (match_operand:DI 1 "s_register_operand" "w")
2909 (match_operand:SI 2 "immediate_operand" "i")]
d98a3884 2910 "TARGET_NEON"
2911{
8521669a 2912 neon_lane_bounds (operands[2], 0, 1);
d98a3884 2913 emit_move_insn (operands[0], operands[1]);
2914 DONE;
2915})
2916
79a83503 2917; Likewise for v2di, as the DImode second operand has only a single element.
2918(define_expand "neon_vdup_lanev2di"
2919 [(match_operand:V2DI 0 "s_register_operand" "=w")
2920 (match_operand:DI 1 "s_register_operand" "w")
2921 (match_operand:SI 2 "immediate_operand" "i")]
d98a3884 2922 "TARGET_NEON"
8521669a 2923{
2924 neon_lane_bounds (operands[2], 0, 1);
79a83503 2925 emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2926 DONE;
2927})
d98a3884 2928
47ddcd6b 2929; Disabled before reload because we don't want combine doing something silly,
2930; but used by the post-reload expansion of neon_vcombine.
2931(define_insn "*neon_vswp<mode>"
2932 [(set (match_operand:VDQX 0 "s_register_operand" "+w")
2933 (match_operand:VDQX 1 "s_register_operand" "+w"))
2934 (set (match_dup 1) (match_dup 0))]
2935 "TARGET_NEON && reload_completed"
042161e8 2936 "vswp\t%<V_reg>0, %<V_reg>1"
47ddcd6b 2937 [(set (attr "neon_type")
2938 (if_then_else (match_test "<Is_d_reg>")
2939 (const_string "neon_bp_simple")
2940 (const_string "neon_bp_2cycle")))]
2941)
2942
d98a3884 2943;; In this insn, operand 1 should be low, and operand 2 the high part of the
2944;; dest vector.
2945;; FIXME: A different implementation of this builtin could make it much
2946;; more likely that we wouldn't actually need to output anything (we could make
2947;; it so that the reg allocator puts things in the right places magically
2948;; instead). Lack of subregs for vectors makes that tricky though, I think.
2949
47ddcd6b 2950(define_insn_and_split "neon_vcombine<mode>"
d98a3884 2951 [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
47ddcd6b 2952 (vec_concat:<V_DOUBLE>
2953 (match_operand:VDX 1 "s_register_operand" "w")
2954 (match_operand:VDX 2 "s_register_operand" "w")))]
d98a3884 2955 "TARGET_NEON"
47ddcd6b 2956 "#"
2957 "&& reload_completed"
2958 [(const_int 0)]
d98a3884 2959{
47ddcd6b 2960 neon_split_vcombine (operands);
2961 DONE;
2962})
d98a3884 2963
33aeac4f 2964(define_expand "neon_vget_high<mode>"
2965 [(match_operand:<V_HALF> 0 "s_register_operand")
2966 (match_operand:VQX 1 "s_register_operand")]
79a83503 2967 "TARGET_NEON"
2968{
33aeac4f 2969 emit_move_insn (operands[0],
2970 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
2971 GET_MODE_SIZE (<V_HALF>mode)));
2972 DONE;
2973})
79a83503 2974
33aeac4f 2975(define_expand "neon_vget_low<mode>"
2976 [(match_operand:<V_HALF> 0 "s_register_operand")
2977 (match_operand:VQX 1 "s_register_operand")]
d98a3884 2978 "TARGET_NEON"
2979{
33aeac4f 2980 emit_move_insn (operands[0],
2981 simplify_gen_subreg (<V_HALF>mode, operands[1],
2982 <MODE>mode, 0));
2983 DONE;
2984})
d98a3884 2985
741cdc29 2986(define_insn "float<mode><V_cvtto>2"
2987 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2988 (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
2989 "TARGET_NEON && !flag_rounding_math"
2990 "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
2991 [(set (attr "neon_type")
2992 (if_then_else (match_test "<Is_d_reg>")
2993 (const_string "neon_fp_vadd_ddd_vabs_dd")
2994 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2995)
2996
2997(define_insn "floatuns<mode><V_cvtto>2"
2998 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2999 (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3000 "TARGET_NEON && !flag_rounding_math"
3001 "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3002 [(set (attr "neon_type")
3003 (if_then_else (match_test "<Is_d_reg>")
3004 (const_string "neon_fp_vadd_ddd_vabs_dd")
3005 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3006)
3007
3008(define_insn "fix_trunc<mode><V_cvtto>2"
3009 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3010 (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3011 "TARGET_NEON"
3012 "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3013 [(set (attr "neon_type")
3014 (if_then_else (match_test "<Is_d_reg>")
3015 (const_string "neon_fp_vadd_ddd_vabs_dd")
3016 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3017)
3018
3019(define_insn "fixuns_trunc<mode><V_cvtto>2"
3020 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3021 (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3022 "TARGET_NEON"
3023 "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3024 [(set (attr "neon_type")
3025 (if_then_else (match_test "<Is_d_reg>")
3026 (const_string "neon_fp_vadd_ddd_vabs_dd")
3027 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3028)
3029
d98a3884 3030(define_insn "neon_vcvt<mode>"
3031 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3032 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3033 (match_operand:SI 2 "immediate_operand" "i")]
3034 UNSPEC_VCVT))]
3035 "TARGET_NEON"
bcaec148 3036 "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3037 [(set (attr "neon_type")
0bf497f5 3038 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3039 (const_string "neon_fp_vadd_ddd_vabs_dd")
3040 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3041)
d98a3884 3042
3043(define_insn "neon_vcvt<mode>"
3044 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3045 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3046 (match_operand:SI 2 "immediate_operand" "i")]
3047 UNSPEC_VCVT))]
3048 "TARGET_NEON"
bcaec148 3049 "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3050 [(set (attr "neon_type")
0bf497f5 3051 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3052 (const_string "neon_fp_vadd_ddd_vabs_dd")
3053 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3054)
d98a3884 3055
3056(define_insn "neon_vcvt_n<mode>"
3057 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3058 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3059 (match_operand:SI 2 "immediate_operand" "i")
3060 (match_operand:SI 3 "immediate_operand" "i")]
3061 UNSPEC_VCVT_N))]
3062 "TARGET_NEON"
8521669a 3063{
3064 neon_const_bounds (operands[2], 1, 33);
3065 return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3066}
bcaec148 3067 [(set (attr "neon_type")
0bf497f5 3068 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3069 (const_string "neon_fp_vadd_ddd_vabs_dd")
3070 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3071)
d98a3884 3072
3073(define_insn "neon_vcvt_n<mode>"
3074 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3075 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3076 (match_operand:SI 2 "immediate_operand" "i")
3077 (match_operand:SI 3 "immediate_operand" "i")]
3078 UNSPEC_VCVT_N))]
3079 "TARGET_NEON"
8521669a 3080{
3081 neon_const_bounds (operands[2], 1, 33);
3082 return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3083}
bcaec148 3084 [(set (attr "neon_type")
0bf497f5 3085 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3086 (const_string "neon_fp_vadd_ddd_vabs_dd")
3087 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3088)
d98a3884 3089
3090(define_insn "neon_vmovn<mode>"
3091 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3092 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3093 (match_operand:SI 2 "immediate_operand" "i")]
3094 UNSPEC_VMOVN))]
3095 "TARGET_NEON"
bcaec148 3096 "vmovn.<V_if_elem>\t%P0, %q1"
3097 [(set_attr "neon_type" "neon_bp_simple")]
3098)
d98a3884 3099
3100(define_insn "neon_vqmovn<mode>"
3101 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3102 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3103 (match_operand:SI 2 "immediate_operand" "i")]
3104 UNSPEC_VQMOVN))]
3105 "TARGET_NEON"
bcaec148 3106 "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3107 [(set_attr "neon_type" "neon_shift_2")]
3108)
d98a3884 3109
3110(define_insn "neon_vqmovun<mode>"
3111 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3112 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3113 (match_operand:SI 2 "immediate_operand" "i")]
3114 UNSPEC_VQMOVUN))]
3115 "TARGET_NEON"
bcaec148 3116 "vqmovun.<V_s_elem>\t%P0, %q1"
3117 [(set_attr "neon_type" "neon_shift_2")]
3118)
d98a3884 3119
3120(define_insn "neon_vmovl<mode>"
3121 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3122 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3123 (match_operand:SI 2 "immediate_operand" "i")]
3124 UNSPEC_VMOVL))]
3125 "TARGET_NEON"
bcaec148 3126 "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3127 [(set_attr "neon_type" "neon_shift_1")]
3128)
d98a3884 3129
3130(define_insn "neon_vmul_lane<mode>"
3131 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3132 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3133 (match_operand:VMD 2 "s_register_operand"
3134 "<scalar_mul_constraint>")
3135 (match_operand:SI 3 "immediate_operand" "i")
3136 (match_operand:SI 4 "immediate_operand" "i")]
3137 UNSPEC_VMUL_LANE))]
3138 "TARGET_NEON"
8521669a 3139{
3140 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3141 return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3142}
bcaec148 3143 [(set (attr "neon_type")
0bf497f5 3144 (if_then_else (match_test "<Is_float_mode>")
bcaec148 3145 (const_string "neon_fp_vmul_ddd")
0bf497f5 3146 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3147 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3148 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3149)
d98a3884 3150
3151(define_insn "neon_vmul_lane<mode>"
3152 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3153 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3154 (match_operand:<V_HALF> 2 "s_register_operand"
3155 "<scalar_mul_constraint>")
3156 (match_operand:SI 3 "immediate_operand" "i")
3157 (match_operand:SI 4 "immediate_operand" "i")]
3158 UNSPEC_VMUL_LANE))]
3159 "TARGET_NEON"
8521669a 3160{
3161 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3162 return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3163}
bcaec148 3164 [(set (attr "neon_type")
0bf497f5 3165 (if_then_else (match_test "<Is_float_mode>")
bcaec148 3166 (const_string "neon_fp_vmul_qqd")
0bf497f5 3167 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3168 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3169 (const_string "neon_mul_qqd_32_scalar"))))]
3170)
d98a3884 3171
3172(define_insn "neon_vmull_lane<mode>"
3173 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3174 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3175 (match_operand:VMDI 2 "s_register_operand"
3176 "<scalar_mul_constraint>")
3177 (match_operand:SI 3 "immediate_operand" "i")
3178 (match_operand:SI 4 "immediate_operand" "i")]
3179 UNSPEC_VMULL_LANE))]
3180 "TARGET_NEON"
8521669a 3181{
3182 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3183 return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3184}
bcaec148 3185 [(set (attr "neon_type")
0bf497f5 3186 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3187 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3188 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3189)
d98a3884 3190
3191(define_insn "neon_vqdmull_lane<mode>"
3192 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3193 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3194 (match_operand:VMDI 2 "s_register_operand"
3195 "<scalar_mul_constraint>")
3196 (match_operand:SI 3 "immediate_operand" "i")
3197 (match_operand:SI 4 "immediate_operand" "i")]
3198 UNSPEC_VQDMULL_LANE))]
3199 "TARGET_NEON"
8521669a 3200{
3201 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3202 return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3203}
bcaec148 3204 [(set (attr "neon_type")
0bf497f5 3205 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3206 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3207 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3208)
d98a3884 3209
3210(define_insn "neon_vqdmulh_lane<mode>"
3211 [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3212 (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3213 (match_operand:<V_HALF> 2 "s_register_operand"
3214 "<scalar_mul_constraint>")
3215 (match_operand:SI 3 "immediate_operand" "i")
3216 (match_operand:SI 4 "immediate_operand" "i")]
3217 UNSPEC_VQDMULH_LANE))]
3218 "TARGET_NEON"
8521669a 3219{
3220 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3221 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3222}
bcaec148 3223 [(set (attr "neon_type")
0bf497f5 3224 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3225 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3226 (const_string "neon_mul_qqd_32_scalar")))]
3227)
d98a3884 3228
3229(define_insn "neon_vqdmulh_lane<mode>"
3230 [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3231 (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3232 (match_operand:VMDI 2 "s_register_operand"
3233 "<scalar_mul_constraint>")
3234 (match_operand:SI 3 "immediate_operand" "i")
3235 (match_operand:SI 4 "immediate_operand" "i")]
3236 UNSPEC_VQDMULH_LANE))]
3237 "TARGET_NEON"
8521669a 3238{
3239 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3240 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3241}
bcaec148 3242 [(set (attr "neon_type")
0bf497f5 3243 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3244 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3245 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3246)
d98a3884 3247
3248(define_insn "neon_vmla_lane<mode>"
3249 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3250 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3251 (match_operand:VMD 2 "s_register_operand" "w")
3252 (match_operand:VMD 3 "s_register_operand"
3253 "<scalar_mul_constraint>")
3254 (match_operand:SI 4 "immediate_operand" "i")
3255 (match_operand:SI 5 "immediate_operand" "i")]
3256 UNSPEC_VMLA_LANE))]
3257 "TARGET_NEON"
8521669a 3258{
3259 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3260 return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3261}
bcaec148 3262 [(set (attr "neon_type")
0bf497f5 3263 (if_then_else (match_test "<Is_float_mode>")
bcaec148 3264 (const_string "neon_fp_vmla_ddd_scalar")
0bf497f5 3265 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3266 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3267 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3268)
d98a3884 3269
3270(define_insn "neon_vmla_lane<mode>"
3271 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3272 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3273 (match_operand:VMQ 2 "s_register_operand" "w")
3274 (match_operand:<V_HALF> 3 "s_register_operand"
3275 "<scalar_mul_constraint>")
3276 (match_operand:SI 4 "immediate_operand" "i")
3277 (match_operand:SI 5 "immediate_operand" "i")]
3278 UNSPEC_VMLA_LANE))]
3279 "TARGET_NEON"
8521669a 3280{
3281 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3282 return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3283}
bcaec148 3284 [(set (attr "neon_type")
0bf497f5 3285 (if_then_else (match_test "<Is_float_mode>")
bcaec148 3286 (const_string "neon_fp_vmla_qqq_scalar")
0bf497f5 3287 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3288 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3289 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3290)
d98a3884 3291
3292(define_insn "neon_vmlal_lane<mode>"
3293 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3294 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3295 (match_operand:VMDI 2 "s_register_operand" "w")
3296 (match_operand:VMDI 3 "s_register_operand"
3297 "<scalar_mul_constraint>")
3298 (match_operand:SI 4 "immediate_operand" "i")
3299 (match_operand:SI 5 "immediate_operand" "i")]
3300 UNSPEC_VMLAL_LANE))]
3301 "TARGET_NEON"
8521669a 3302{
3303 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3304 return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3305}
bcaec148 3306 [(set (attr "neon_type")
0bf497f5 3307 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3308 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3309 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3310)
d98a3884 3311
3312(define_insn "neon_vqdmlal_lane<mode>"
3313 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3314 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3315 (match_operand:VMDI 2 "s_register_operand" "w")
3316 (match_operand:VMDI 3 "s_register_operand"
3317 "<scalar_mul_constraint>")
3318 (match_operand:SI 4 "immediate_operand" "i")
3319 (match_operand:SI 5 "immediate_operand" "i")]
3320 UNSPEC_VQDMLAL_LANE))]
3321 "TARGET_NEON"
8521669a 3322{
3323 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3324 return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3325}
bcaec148 3326 [(set (attr "neon_type")
0bf497f5 3327 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3328 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3329 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3330)
d98a3884 3331
3332(define_insn "neon_vmls_lane<mode>"
3333 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3334 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3335 (match_operand:VMD 2 "s_register_operand" "w")
3336 (match_operand:VMD 3 "s_register_operand"
3337 "<scalar_mul_constraint>")
3338 (match_operand:SI 4 "immediate_operand" "i")
3339 (match_operand:SI 5 "immediate_operand" "i")]
3340 UNSPEC_VMLS_LANE))]
3341 "TARGET_NEON"
8521669a 3342{
3343 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3344 return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3345}
bcaec148 3346 [(set (attr "neon_type")
0bf497f5 3347 (if_then_else (match_test "<Is_float_mode>")
bcaec148 3348 (const_string "neon_fp_vmla_ddd_scalar")
0bf497f5 3349 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3350 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3351 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3352)
d98a3884 3353
3354(define_insn "neon_vmls_lane<mode>"
3355 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3356 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3357 (match_operand:VMQ 2 "s_register_operand" "w")
3358 (match_operand:<V_HALF> 3 "s_register_operand"
3359 "<scalar_mul_constraint>")
3360 (match_operand:SI 4 "immediate_operand" "i")
3361 (match_operand:SI 5 "immediate_operand" "i")]
3362 UNSPEC_VMLS_LANE))]
3363 "TARGET_NEON"
8521669a 3364{
3365 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3366 return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3367}
bcaec148 3368 [(set (attr "neon_type")
0bf497f5 3369 (if_then_else (match_test "<Is_float_mode>")
bcaec148 3370 (const_string "neon_fp_vmla_qqq_scalar")
0bf497f5 3371 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3372 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3373 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3374)
d98a3884 3375
3376(define_insn "neon_vmlsl_lane<mode>"
3377 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3378 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3379 (match_operand:VMDI 2 "s_register_operand" "w")
3380 (match_operand:VMDI 3 "s_register_operand"
3381 "<scalar_mul_constraint>")
3382 (match_operand:SI 4 "immediate_operand" "i")
3383 (match_operand:SI 5 "immediate_operand" "i")]
3384 UNSPEC_VMLSL_LANE))]
3385 "TARGET_NEON"
8521669a 3386{
3387 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3388 return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3389}
bcaec148 3390 [(set (attr "neon_type")
0bf497f5 3391 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3392 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3393 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3394)
d98a3884 3395
3396(define_insn "neon_vqdmlsl_lane<mode>"
3397 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3398 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3399 (match_operand:VMDI 2 "s_register_operand" "w")
3400 (match_operand:VMDI 3 "s_register_operand"
3401 "<scalar_mul_constraint>")
3402 (match_operand:SI 4 "immediate_operand" "i")
3403 (match_operand:SI 5 "immediate_operand" "i")]
3404 UNSPEC_VQDMLSL_LANE))]
3405 "TARGET_NEON"
8521669a 3406{
3407 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3408 return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3409}
bcaec148 3410 [(set (attr "neon_type")
0bf497f5 3411 (if_then_else (match_test "<Scalar_mul_8_16>")
bcaec148 3412 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3413 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3414)
d98a3884 3415
3416; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3417; core register into a temp register, then use a scalar taken from that. This
3418; isn't an optimal solution if e.g. the scalar has just been read from memory
3419; or extracted from another vector. The latter case it's currently better to
3420; use the "_lane" variant, and the former case can probably be implemented
3421; using vld1_lane, but that hasn't been done yet.
3422
3423(define_expand "neon_vmul_n<mode>"
3424 [(match_operand:VMD 0 "s_register_operand" "")
3425 (match_operand:VMD 1 "s_register_operand" "")
3426 (match_operand:<V_elem> 2 "s_register_operand" "")
3427 (match_operand:SI 3 "immediate_operand" "")]
3428 "TARGET_NEON"
3429{
3430 rtx tmp = gen_reg_rtx (<MODE>mode);
3431 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3432 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3433 const0_rtx, const0_rtx));
3434 DONE;
3435})
3436
3437(define_expand "neon_vmul_n<mode>"
3438 [(match_operand:VMQ 0 "s_register_operand" "")
3439 (match_operand:VMQ 1 "s_register_operand" "")
3440 (match_operand:<V_elem> 2 "s_register_operand" "")
3441 (match_operand:SI 3 "immediate_operand" "")]
3442 "TARGET_NEON"
3443{
3444 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3445 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3446 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3447 const0_rtx, const0_rtx));
3448 DONE;
3449})
3450
3451(define_expand "neon_vmull_n<mode>"
3452 [(match_operand:<V_widen> 0 "s_register_operand" "")
3453 (match_operand:VMDI 1 "s_register_operand" "")
3454 (match_operand:<V_elem> 2 "s_register_operand" "")
3455 (match_operand:SI 3 "immediate_operand" "")]
3456 "TARGET_NEON"
3457{
3458 rtx tmp = gen_reg_rtx (<MODE>mode);
3459 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3460 emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3461 const0_rtx, operands[3]));
3462 DONE;
3463})
3464
3465(define_expand "neon_vqdmull_n<mode>"
3466 [(match_operand:<V_widen> 0 "s_register_operand" "")
3467 (match_operand:VMDI 1 "s_register_operand" "")
3468 (match_operand:<V_elem> 2 "s_register_operand" "")
3469 (match_operand:SI 3 "immediate_operand" "")]
3470 "TARGET_NEON"
3471{
3472 rtx tmp = gen_reg_rtx (<MODE>mode);
3473 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3474 emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3475 const0_rtx, const0_rtx));
3476 DONE;
3477})
3478
3479(define_expand "neon_vqdmulh_n<mode>"
3480 [(match_operand:VMDI 0 "s_register_operand" "")
3481 (match_operand:VMDI 1 "s_register_operand" "")
3482 (match_operand:<V_elem> 2 "s_register_operand" "")
3483 (match_operand:SI 3 "immediate_operand" "")]
3484 "TARGET_NEON"
3485{
3486 rtx tmp = gen_reg_rtx (<MODE>mode);
3487 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3488 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3489 const0_rtx, operands[3]));
3490 DONE;
3491})
3492
3493(define_expand "neon_vqdmulh_n<mode>"
3494 [(match_operand:VMQI 0 "s_register_operand" "")
3495 (match_operand:VMQI 1 "s_register_operand" "")
3496 (match_operand:<V_elem> 2 "s_register_operand" "")
3497 (match_operand:SI 3 "immediate_operand" "")]
3498 "TARGET_NEON"
3499{
3500 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3501 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3502 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3503 const0_rtx, operands[3]));
3504 DONE;
3505})
3506
3507(define_expand "neon_vmla_n<mode>"
3508 [(match_operand:VMD 0 "s_register_operand" "")
3509 (match_operand:VMD 1 "s_register_operand" "")
3510 (match_operand:VMD 2 "s_register_operand" "")
3511 (match_operand:<V_elem> 3 "s_register_operand" "")
3512 (match_operand:SI 4 "immediate_operand" "")]
3513 "TARGET_NEON"
3514{
3515 rtx tmp = gen_reg_rtx (<MODE>mode);
3516 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3517 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3518 tmp, const0_rtx, operands[4]));
3519 DONE;
3520})
3521
3522(define_expand "neon_vmla_n<mode>"
3523 [(match_operand:VMQ 0 "s_register_operand" "")
3524 (match_operand:VMQ 1 "s_register_operand" "")
3525 (match_operand:VMQ 2 "s_register_operand" "")
3526 (match_operand:<V_elem> 3 "s_register_operand" "")
3527 (match_operand:SI 4 "immediate_operand" "")]
3528 "TARGET_NEON"
3529{
3530 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3531 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3532 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3533 tmp, const0_rtx, operands[4]));
3534 DONE;
3535})
3536
3537(define_expand "neon_vmlal_n<mode>"
3538 [(match_operand:<V_widen> 0 "s_register_operand" "")
3539 (match_operand:<V_widen> 1 "s_register_operand" "")
3540 (match_operand:VMDI 2 "s_register_operand" "")
3541 (match_operand:<V_elem> 3 "s_register_operand" "")
3542 (match_operand:SI 4 "immediate_operand" "")]
3543 "TARGET_NEON"
3544{
3545 rtx tmp = gen_reg_rtx (<MODE>mode);
3546 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3547 emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3548 tmp, const0_rtx, operands[4]));
3549 DONE;
3550})
3551
3552(define_expand "neon_vqdmlal_n<mode>"
3553 [(match_operand:<V_widen> 0 "s_register_operand" "")
3554 (match_operand:<V_widen> 1 "s_register_operand" "")
3555 (match_operand:VMDI 2 "s_register_operand" "")
3556 (match_operand:<V_elem> 3 "s_register_operand" "")
3557 (match_operand:SI 4 "immediate_operand" "")]
3558 "TARGET_NEON"
3559{
3560 rtx tmp = gen_reg_rtx (<MODE>mode);
3561 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3562 emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3563 tmp, const0_rtx, operands[4]));
3564 DONE;
3565})
3566
3567(define_expand "neon_vmls_n<mode>"
3568 [(match_operand:VMD 0 "s_register_operand" "")
3569 (match_operand:VMD 1 "s_register_operand" "")
3570 (match_operand:VMD 2 "s_register_operand" "")
3571 (match_operand:<V_elem> 3 "s_register_operand" "")
3572 (match_operand:SI 4 "immediate_operand" "")]
3573 "TARGET_NEON"
3574{
3575 rtx tmp = gen_reg_rtx (<MODE>mode);
3576 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3577 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3578 tmp, const0_rtx, operands[4]));
3579 DONE;
3580})
3581
3582(define_expand "neon_vmls_n<mode>"
3583 [(match_operand:VMQ 0 "s_register_operand" "")
3584 (match_operand:VMQ 1 "s_register_operand" "")
3585 (match_operand:VMQ 2 "s_register_operand" "")
3586 (match_operand:<V_elem> 3 "s_register_operand" "")
3587 (match_operand:SI 4 "immediate_operand" "")]
3588 "TARGET_NEON"
3589{
3590 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3591 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3592 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3593 tmp, const0_rtx, operands[4]));
3594 DONE;
3595})
3596
3597(define_expand "neon_vmlsl_n<mode>"
3598 [(match_operand:<V_widen> 0 "s_register_operand" "")
3599 (match_operand:<V_widen> 1 "s_register_operand" "")
3600 (match_operand:VMDI 2 "s_register_operand" "")
3601 (match_operand:<V_elem> 3 "s_register_operand" "")
3602 (match_operand:SI 4 "immediate_operand" "")]
3603 "TARGET_NEON"
3604{
3605 rtx tmp = gen_reg_rtx (<MODE>mode);
3606 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3607 emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3608 tmp, const0_rtx, operands[4]));
3609 DONE;
3610})
3611
3612(define_expand "neon_vqdmlsl_n<mode>"
3613 [(match_operand:<V_widen> 0 "s_register_operand" "")
3614 (match_operand:<V_widen> 1 "s_register_operand" "")
3615 (match_operand:VMDI 2 "s_register_operand" "")
3616 (match_operand:<V_elem> 3 "s_register_operand" "")
3617 (match_operand:SI 4 "immediate_operand" "")]
3618 "TARGET_NEON"
3619{
3620 rtx tmp = gen_reg_rtx (<MODE>mode);
3621 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3622 emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3623 tmp, const0_rtx, operands[4]));
3624 DONE;
3625})
3626
3627(define_insn "neon_vext<mode>"
3628 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3629 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3630 (match_operand:VDQX 2 "s_register_operand" "w")
3631 (match_operand:SI 3 "immediate_operand" "i")]
3632 UNSPEC_VEXT))]
3633 "TARGET_NEON"
8521669a 3634{
3635 neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3636 return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3637}
bcaec148 3638 [(set (attr "neon_type")
0bf497f5 3639 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3640 (const_string "neon_bp_simple")
3641 (const_string "neon_bp_2cycle")))]
3642)
d98a3884 3643
3644(define_insn "neon_vrev64<mode>"
3645 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3646 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3647 (match_operand:SI 2 "immediate_operand" "i")]
3648 UNSPEC_VREV64))]
3649 "TARGET_NEON"
bcaec148 3650 "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3651 [(set_attr "neon_type" "neon_bp_simple")]
3652)
d98a3884 3653
3654(define_insn "neon_vrev32<mode>"
3655 [(set (match_operand:VX 0 "s_register_operand" "=w")
3656 (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3657 (match_operand:SI 2 "immediate_operand" "i")]
3658 UNSPEC_VREV32))]
3659 "TARGET_NEON"
bcaec148 3660 "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3661 [(set_attr "neon_type" "neon_bp_simple")]
3662)
d98a3884 3663
3664(define_insn "neon_vrev16<mode>"
3665 [(set (match_operand:VE 0 "s_register_operand" "=w")
3666 (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3667 (match_operand:SI 2 "immediate_operand" "i")]
3668 UNSPEC_VREV16))]
3669 "TARGET_NEON"
bcaec148 3670 "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3671 [(set_attr "neon_type" "neon_bp_simple")]
3672)
d98a3884 3673
3674; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3675; allocation. For an intrinsic of form:
3676; rD = vbsl_* (rS, rN, rM)
3677; We can use any of:
3678; vbsl rS, rN, rM (if D = S)
3679; vbit rD, rN, rS (if D = M, so 1-bits in rS choose bits from rN, else rM)
3680; vbif rD, rM, rS (if D = N, so 0-bits in rS choose bits from rM, else rN)
3681
3682(define_insn "neon_vbsl<mode>_internal"
3683 [(set (match_operand:VDQX 0 "s_register_operand" "=w,w,w")
3684 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3685 (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3686 (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3687 UNSPEC_VBSL))]
3688 "TARGET_NEON"
3689 "@
3690 vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3691 vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
bcaec148 3692 vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3693 [(set_attr "neon_type" "neon_int_1")]
3694)
d98a3884 3695
3696(define_expand "neon_vbsl<mode>"
3697 [(set (match_operand:VDQX 0 "s_register_operand" "")
3698 (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3699 (match_operand:VDQX 2 "s_register_operand" "")
3700 (match_operand:VDQX 3 "s_register_operand" "")]
3701 UNSPEC_VBSL))]
3702 "TARGET_NEON"
3703{
3704 /* We can't alias operands together if they have different modes. */
3705 operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3706})
3707
3708(define_insn "neon_vshl<mode>"
3709 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3710 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3711 (match_operand:VDQIX 2 "s_register_operand" "w")
3712 (match_operand:SI 3 "immediate_operand" "i")]
3713 UNSPEC_VSHL))]
3714 "TARGET_NEON"
bcaec148 3715 "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3716 [(set (attr "neon_type")
0bf497f5 3717 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3718 (const_string "neon_vshl_ddd")
3719 (const_string "neon_shift_3")))]
3720)
d98a3884 3721
3722(define_insn "neon_vqshl<mode>"
3723 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3724 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3725 (match_operand:VDQIX 2 "s_register_operand" "w")
3726 (match_operand:SI 3 "immediate_operand" "i")]
3727 UNSPEC_VQSHL))]
3728 "TARGET_NEON"
bcaec148 3729 "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3730 [(set (attr "neon_type")
0bf497f5 3731 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3732 (const_string "neon_shift_2")
3733 (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3734)
d98a3884 3735
3736(define_insn "neon_vshr_n<mode>"
3737 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3738 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3739 (match_operand:SI 2 "immediate_operand" "i")
3740 (match_operand:SI 3 "immediate_operand" "i")]
3741 UNSPEC_VSHR_N))]
3742 "TARGET_NEON"
8521669a 3743{
3744 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3745 return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3746}
bcaec148 3747 [(set_attr "neon_type" "neon_shift_1")]
3748)
d98a3884 3749
3750(define_insn "neon_vshrn_n<mode>"
3751 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3752 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3753 (match_operand:SI 2 "immediate_operand" "i")
3754 (match_operand:SI 3 "immediate_operand" "i")]
3755 UNSPEC_VSHRN_N))]
3756 "TARGET_NEON"
8521669a 3757{
3758 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3759 return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3760}
bcaec148 3761 [(set_attr "neon_type" "neon_shift_1")]
3762)
d98a3884 3763
3764(define_insn "neon_vqshrn_n<mode>"
3765 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3766 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3767 (match_operand:SI 2 "immediate_operand" "i")
3768 (match_operand:SI 3 "immediate_operand" "i")]
3769 UNSPEC_VQSHRN_N))]
3770 "TARGET_NEON"
8521669a 3771{
3772 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3773 return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3774}
bcaec148 3775 [(set_attr "neon_type" "neon_shift_2")]
3776)
d98a3884 3777
3778(define_insn "neon_vqshrun_n<mode>"
3779 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3780 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3781 (match_operand:SI 2 "immediate_operand" "i")
3782 (match_operand:SI 3 "immediate_operand" "i")]
3783 UNSPEC_VQSHRUN_N))]
3784 "TARGET_NEON"
8521669a 3785{
3786 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3787 return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3788}
bcaec148 3789 [(set_attr "neon_type" "neon_shift_2")]
3790)
d98a3884 3791
3792(define_insn "neon_vshl_n<mode>"
3793 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3794 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3795 (match_operand:SI 2 "immediate_operand" "i")
3796 (match_operand:SI 3 "immediate_operand" "i")]
3797 UNSPEC_VSHL_N))]
3798 "TARGET_NEON"
8521669a 3799{
3800 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3801 return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3802}
bcaec148 3803 [(set_attr "neon_type" "neon_shift_1")]
3804)
d98a3884 3805
3806(define_insn "neon_vqshl_n<mode>"
3807 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3808 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3809 (match_operand:SI 2 "immediate_operand" "i")
3810 (match_operand:SI 3 "immediate_operand" "i")]
3811 UNSPEC_VQSHL_N))]
3812 "TARGET_NEON"
8521669a 3813{
3814 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3815 return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3816}
bcaec148 3817 [(set_attr "neon_type" "neon_shift_2")]
3818)
d98a3884 3819
3820(define_insn "neon_vqshlu_n<mode>"
3821 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3822 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3823 (match_operand:SI 2 "immediate_operand" "i")
3824 (match_operand:SI 3 "immediate_operand" "i")]
3825 UNSPEC_VQSHLU_N))]
3826 "TARGET_NEON"
8521669a 3827{
3828 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3829 return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3830}
bcaec148 3831 [(set_attr "neon_type" "neon_shift_2")]
3832)
d98a3884 3833
3834(define_insn "neon_vshll_n<mode>"
3835 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3836 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3837 (match_operand:SI 2 "immediate_operand" "i")
3838 (match_operand:SI 3 "immediate_operand" "i")]
3839 UNSPEC_VSHLL_N))]
3840 "TARGET_NEON"
8521669a 3841{
61725b11 3842 /* The boundaries are: 0 < imm <= size. */
3843 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
8521669a 3844 return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3845}
bcaec148 3846 [(set_attr "neon_type" "neon_shift_1")]
3847)
d98a3884 3848
3849(define_insn "neon_vsra_n<mode>"
3850 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3851 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3852 (match_operand:VDQIX 2 "s_register_operand" "w")
3853 (match_operand:SI 3 "immediate_operand" "i")
3854 (match_operand:SI 4 "immediate_operand" "i")]
3855 UNSPEC_VSRA_N))]
3856 "TARGET_NEON"
8521669a 3857{
3858 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3859 return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3860}
bcaec148 3861 [(set_attr "neon_type" "neon_vsra_vrsra")]
3862)
d98a3884 3863
3864(define_insn "neon_vsri_n<mode>"
3865 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3866 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3867 (match_operand:VDQIX 2 "s_register_operand" "w")
3868 (match_operand:SI 3 "immediate_operand" "i")]
3869 UNSPEC_VSRI))]
3870 "TARGET_NEON"
8521669a 3871{
3872 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3873 return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3874}
bcaec148 3875 [(set (attr "neon_type")
0bf497f5 3876 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3877 (const_string "neon_shift_1")
3878 (const_string "neon_shift_3")))]
3879)
d98a3884 3880
3881(define_insn "neon_vsli_n<mode>"
3882 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3883 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3884 (match_operand:VDQIX 2 "s_register_operand" "w")
3885 (match_operand:SI 3 "immediate_operand" "i")]
3886 UNSPEC_VSLI))]
3887 "TARGET_NEON"
8521669a 3888{
3889 neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3890 return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3891}
bcaec148 3892 [(set (attr "neon_type")
0bf497f5 3893 (if_then_else (match_test "<Is_d_reg>")
bcaec148 3894 (const_string "neon_shift_1")
3895 (const_string "neon_shift_3")))]
3896)
d98a3884 3897
3898(define_insn "neon_vtbl1v8qi"
3899 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3900 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3901 (match_operand:V8QI 2 "s_register_operand" "w")]
3902 UNSPEC_VTBL))]
3903 "TARGET_NEON"
bcaec148 3904 "vtbl.8\t%P0, {%P1}, %P2"
3905 [(set_attr "neon_type" "neon_bp_2cycle")]
3906)
d98a3884 3907
3908(define_insn "neon_vtbl2v8qi"
3909 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3910 (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3911 (match_operand:V8QI 2 "s_register_operand" "w")]
3912 UNSPEC_VTBL))]
3913 "TARGET_NEON"
3914{
3915 rtx ops[4];
3916 int tabbase = REGNO (operands[1]);
3917
3918 ops[0] = operands[0];
3919 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3920 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3921 ops[3] = operands[2];
3922 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3923
3924 return "";
bcaec148 3925}
3926 [(set_attr "neon_type" "neon_bp_2cycle")]
3927)
d98a3884 3928
3929(define_insn "neon_vtbl3v8qi"
3930 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3931 (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3932 (match_operand:V8QI 2 "s_register_operand" "w")]
3933 UNSPEC_VTBL))]
3934 "TARGET_NEON"
3935{
3936 rtx ops[5];
3937 int tabbase = REGNO (operands[1]);
3938
3939 ops[0] = operands[0];
3940 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3941 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3942 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3943 ops[4] = operands[2];
3944 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3945
3946 return "";
bcaec148 3947}
3948 [(set_attr "neon_type" "neon_bp_3cycle")]
3949)
d98a3884 3950
3951(define_insn "neon_vtbl4v8qi"
3952 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3953 (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3954 (match_operand:V8QI 2 "s_register_operand" "w")]
3955 UNSPEC_VTBL))]
3956 "TARGET_NEON"
3957{
3958 rtx ops[6];
3959 int tabbase = REGNO (operands[1]);
3960
3961 ops[0] = operands[0];
3962 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3963 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3964 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3965 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
3966 ops[5] = operands[2];
3967 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
3968
3969 return "";
bcaec148 3970}
3971 [(set_attr "neon_type" "neon_bp_3cycle")]
3972)
d98a3884 3973
47ddcd6b 3974;; These three are used by the vec_perm infrastructure for V16QImode.
3975(define_insn_and_split "neon_vtbl1v16qi"
3976 [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3977 (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
3978 (match_operand:V16QI 2 "s_register_operand" "w")]
3979 UNSPEC_VTBL))]
3980 "TARGET_NEON"
3981 "#"
3982 "&& reload_completed"
3983 [(const_int 0)]
3984{
3985 rtx op0, op1, op2, part0, part2;
3986 unsigned ofs;
3987
3988 op0 = operands[0];
3989 op1 = gen_lowpart (TImode, operands[1]);
3990 op2 = operands[2];
3991
3992 ofs = subreg_lowpart_offset (V8QImode, V16QImode);
3993 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3994 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3995 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3996
3997 ofs = subreg_highpart_offset (V8QImode, V16QImode);
3998 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3999 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4000 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4001 DONE;
4002})
4003
4004(define_insn_and_split "neon_vtbl2v16qi"
4005 [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4006 (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4007 (match_operand:V16QI 2 "s_register_operand" "w")]
4008 UNSPEC_VTBL))]
4009 "TARGET_NEON"
4010 "#"
4011 "&& reload_completed"
4012 [(const_int 0)]
4013{
4014 rtx op0, op1, op2, part0, part2;
4015 unsigned ofs;
4016
4017 op0 = operands[0];
4018 op1 = operands[1];
4019 op2 = operands[2];
4020
4021 ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4022 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4023 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4024 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4025
4026 ofs = subreg_highpart_offset (V8QImode, V16QImode);
4027 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4028 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4029 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4030 DONE;
4031})
4032
4033;; ??? Logically we should extend the regular neon_vcombine pattern to
4034;; handle quad-word input modes, producing octa-word output modes. But
4035;; that requires us to add support for octa-word vector modes in moves.
4036;; That seems overkill for this one use in vec_perm.
4037(define_insn_and_split "neon_vcombinev16qi"
4038 [(set (match_operand:OI 0 "s_register_operand" "=w")
4039 (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4040 (match_operand:V16QI 2 "s_register_operand" "w")]
4041 UNSPEC_VCONCAT))]
4042 "TARGET_NEON"
4043 "#"
4044 "&& reload_completed"
4045 [(const_int 0)]
4046{
4047 neon_split_vcombine (operands);
4048 DONE;
4049})
4050
d98a3884 4051(define_insn "neon_vtbx1v8qi"
4052 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4053 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4054 (match_operand:V8QI 2 "s_register_operand" "w")
4055 (match_operand:V8QI 3 "s_register_operand" "w")]
4056 UNSPEC_VTBX))]
4057 "TARGET_NEON"
bcaec148 4058 "vtbx.8\t%P0, {%P2}, %P3"
4059 [(set_attr "neon_type" "neon_bp_2cycle")]
4060)
d98a3884 4061
4062(define_insn "neon_vtbx2v8qi"
4063 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4064 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4065 (match_operand:TI 2 "s_register_operand" "w")
4066 (match_operand:V8QI 3 "s_register_operand" "w")]
4067 UNSPEC_VTBX))]
4068 "TARGET_NEON"
4069{
4070 rtx ops[4];
4071 int tabbase = REGNO (operands[2]);
4072
4073 ops[0] = operands[0];
4074 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4075 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4076 ops[3] = operands[3];
4077 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4078
4079 return "";
bcaec148 4080}
4081 [(set_attr "neon_type" "neon_bp_2cycle")]
4082)
d98a3884 4083
4084(define_insn "neon_vtbx3v8qi"
4085 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4086 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4087 (match_operand:EI 2 "s_register_operand" "w")
4088 (match_operand:V8QI 3 "s_register_operand" "w")]
4089 UNSPEC_VTBX))]
4090 "TARGET_NEON"
4091{
4092 rtx ops[5];
4093 int tabbase = REGNO (operands[2]);
4094
4095 ops[0] = operands[0];
4096 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4097 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4098 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4099 ops[4] = operands[3];
4100 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4101
4102 return "";
bcaec148 4103}
4104 [(set_attr "neon_type" "neon_bp_3cycle")]
4105)
d98a3884 4106
4107(define_insn "neon_vtbx4v8qi"
4108 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4109 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4110 (match_operand:OI 2 "s_register_operand" "w")
4111 (match_operand:V8QI 3 "s_register_operand" "w")]
4112 UNSPEC_VTBX))]
4113 "TARGET_NEON"
4114{
4115 rtx ops[6];
4116 int tabbase = REGNO (operands[2]);
4117
4118 ops[0] = operands[0];
4119 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4120 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4121 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4122 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4123 ops[5] = operands[3];
4124 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4125
4126 return "";
bcaec148 4127}
4128 [(set_attr "neon_type" "neon_bp_3cycle")]
4129)
d98a3884 4130
4131(define_insn "neon_vtrn<mode>_internal"
4132 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
fd4b5409 4133 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4134 (match_operand:VDQW 2 "s_register_operand" "w")]
4135 UNSPEC_VTRN1))
4136 (set (match_operand:VDQW 3 "s_register_operand" "=2")
4137 (unspec:VDQW [(match_dup 1) (match_dup 2)]
4138 UNSPEC_VTRN2))]
4139 "TARGET_NEON"
4140 "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
bcaec148 4141 [(set (attr "neon_type")
0bf497f5 4142 (if_then_else (match_test "<Is_d_reg>")
bcaec148 4143 (const_string "neon_bp_simple")
4144 (const_string "neon_bp_3cycle")))]
4145)
d98a3884 4146
4147(define_expand "neon_vtrn<mode>"
4148 [(match_operand:SI 0 "s_register_operand" "r")
4149 (match_operand:VDQW 1 "s_register_operand" "w")
4150 (match_operand:VDQW 2 "s_register_operand" "w")]
4151 "TARGET_NEON"
4152{
4153 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4154 operands[0], operands[1], operands[2]);
4155 DONE;
4156})
4157
4158(define_insn "neon_vzip<mode>_internal"
4159 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
fd4b5409 4160 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4161 (match_operand:VDQW 2 "s_register_operand" "w")]
4162 UNSPEC_VZIP1))
4163 (set (match_operand:VDQW 3 "s_register_operand" "=2")
4164 (unspec:VDQW [(match_dup 1) (match_dup 2)]
4165 UNSPEC_VZIP2))]
4166 "TARGET_NEON"
4167 "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
bcaec148 4168 [(set (attr "neon_type")
0bf497f5 4169 (if_then_else (match_test "<Is_d_reg>")
bcaec148 4170 (const_string "neon_bp_simple")
4171 (const_string "neon_bp_3cycle")))]
4172)
d98a3884 4173
4174(define_expand "neon_vzip<mode>"
4175 [(match_operand:SI 0 "s_register_operand" "r")
4176 (match_operand:VDQW 1 "s_register_operand" "w")
4177 (match_operand:VDQW 2 "s_register_operand" "w")]
4178 "TARGET_NEON"
4179{
4180 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4181 operands[0], operands[1], operands[2]);
4182 DONE;
4183})
4184
4185(define_insn "neon_vuzp<mode>_internal"
4186 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
fd4b5409 4187 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4188 (match_operand:VDQW 2 "s_register_operand" "w")]
d98a3884 4189 UNSPEC_VUZP1))
fd4b5409 4190 (set (match_operand:VDQW 3 "s_register_operand" "=2")
4191 (unspec:VDQW [(match_dup 1) (match_dup 2)]
4192 UNSPEC_VUZP2))]
d98a3884 4193 "TARGET_NEON"
fd4b5409 4194 "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
bcaec148 4195 [(set (attr "neon_type")
0bf497f5 4196 (if_then_else (match_test "<Is_d_reg>")
bcaec148 4197 (const_string "neon_bp_simple")
4198 (const_string "neon_bp_3cycle")))]
4199)
d98a3884 4200
4201(define_expand "neon_vuzp<mode>"
4202 [(match_operand:SI 0 "s_register_operand" "r")
4203 (match_operand:VDQW 1 "s_register_operand" "w")
4204 (match_operand:VDQW 2 "s_register_operand" "w")]
4205 "TARGET_NEON"
4206{
4207 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4208 operands[0], operands[1], operands[2]);
4209 DONE;
4210})
4211
4212(define_expand "neon_vreinterpretv8qi<mode>"
4213 [(match_operand:V8QI 0 "s_register_operand" "")
4214 (match_operand:VDX 1 "s_register_operand" "")]
4215 "TARGET_NEON"
4216{
4217 neon_reinterpret (operands[0], operands[1]);
4218 DONE;
4219})
4220
4221(define_expand "neon_vreinterpretv4hi<mode>"
4222 [(match_operand:V4HI 0 "s_register_operand" "")
4223 (match_operand:VDX 1 "s_register_operand" "")]
4224 "TARGET_NEON"
4225{
4226 neon_reinterpret (operands[0], operands[1]);
4227 DONE;
4228})
4229
4230(define_expand "neon_vreinterpretv2si<mode>"
4231 [(match_operand:V2SI 0 "s_register_operand" "")
4232 (match_operand:VDX 1 "s_register_operand" "")]
4233 "TARGET_NEON"
4234{
4235 neon_reinterpret (operands[0], operands[1]);
4236 DONE;
4237})
4238
4239(define_expand "neon_vreinterpretv2sf<mode>"
4240 [(match_operand:V2SF 0 "s_register_operand" "")
4241 (match_operand:VDX 1 "s_register_operand" "")]
4242 "TARGET_NEON"
4243{
4244 neon_reinterpret (operands[0], operands[1]);
4245 DONE;
4246})
4247
4248(define_expand "neon_vreinterpretdi<mode>"
4249 [(match_operand:DI 0 "s_register_operand" "")
4250 (match_operand:VDX 1 "s_register_operand" "")]
4251 "TARGET_NEON"
4252{
4253 neon_reinterpret (operands[0], operands[1]);
4254 DONE;
4255})
4256
4257(define_expand "neon_vreinterpretv16qi<mode>"
4258 [(match_operand:V16QI 0 "s_register_operand" "")
4259 (match_operand:VQX 1 "s_register_operand" "")]
4260 "TARGET_NEON"
4261{
4262 neon_reinterpret (operands[0], operands[1]);
4263 DONE;
4264})
4265
4266(define_expand "neon_vreinterpretv8hi<mode>"
4267 [(match_operand:V8HI 0 "s_register_operand" "")
4268 (match_operand:VQX 1 "s_register_operand" "")]
4269 "TARGET_NEON"
4270{
4271 neon_reinterpret (operands[0], operands[1]);
4272 DONE;
4273})
4274
4275(define_expand "neon_vreinterpretv4si<mode>"
4276 [(match_operand:V4SI 0 "s_register_operand" "")
4277 (match_operand:VQX 1 "s_register_operand" "")]
4278 "TARGET_NEON"
4279{
4280 neon_reinterpret (operands[0], operands[1]);
4281 DONE;
4282})
4283
4284(define_expand "neon_vreinterpretv4sf<mode>"
4285 [(match_operand:V4SF 0 "s_register_operand" "")
4286 (match_operand:VQX 1 "s_register_operand" "")]
4287 "TARGET_NEON"
4288{
4289 neon_reinterpret (operands[0], operands[1]);
4290 DONE;
4291})
4292
4293(define_expand "neon_vreinterpretv2di<mode>"
4294 [(match_operand:V2DI 0 "s_register_operand" "")
4295 (match_operand:VQX 1 "s_register_operand" "")]
4296 "TARGET_NEON"
4297{
4298 neon_reinterpret (operands[0], operands[1]);
4299 DONE;
4300})
4301
672b3f5b 4302(define_expand "vec_load_lanes<mode><mode>"
4303 [(set (match_operand:VDQX 0 "s_register_operand")
4304 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4305 UNSPEC_VLD1))]
4306 "TARGET_NEON")
4307
d98a3884 4308(define_insn "neon_vld1<mode>"
4309 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
e5bf7a7a 4310 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
d98a3884 4311 UNSPEC_VLD1))]
4312 "TARGET_NEON"
e5bf7a7a 4313 "vld1.<V_sz_elem>\t%h0, %A1"
bcaec148 4314 [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4315)
d98a3884 4316
4317(define_insn "neon_vld1_lane<mode>"
4318 [(set (match_operand:VDX 0 "s_register_operand" "=w")
e5bf7a7a 4319 (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
d98a3884 4320 (match_operand:VDX 2 "s_register_operand" "0")
4321 (match_operand:SI 3 "immediate_operand" "i")]
4322 UNSPEC_VLD1_LANE))]
4323 "TARGET_NEON"
4324{
4325 HOST_WIDE_INT lane = INTVAL (operands[3]);
4326 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4327 if (lane < 0 || lane >= max)
4328 error ("lane out of range");
4329 if (max == 1)
e5bf7a7a 4330 return "vld1.<V_sz_elem>\t%P0, %A1";
d98a3884 4331 else
e5bf7a7a 4332 return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
bcaec148 4333}
4334 [(set (attr "neon_type")
4335 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4336 (const_string "neon_vld1_1_2_regs")
4337 (const_string "neon_vld1_vld2_lane")))]
4338)
d98a3884 4339
4340(define_insn "neon_vld1_lane<mode>"
4341 [(set (match_operand:VQX 0 "s_register_operand" "=w")
e5bf7a7a 4342 (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
d98a3884 4343 (match_operand:VQX 2 "s_register_operand" "0")
4344 (match_operand:SI 3 "immediate_operand" "i")]
4345 UNSPEC_VLD1_LANE))]
4346 "TARGET_NEON"
4347{
4348 HOST_WIDE_INT lane = INTVAL (operands[3]);
4349 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4350 int regno = REGNO (operands[0]);
4351 if (lane < 0 || lane >= max)
4352 error ("lane out of range");
4353 else if (lane >= max / 2)
4354 {
4355 lane -= max / 2;
4356 regno += 2;
4357 operands[3] = GEN_INT (lane);
4358 }
4359 operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4360 if (max == 2)
e5bf7a7a 4361 return "vld1.<V_sz_elem>\t%P0, %A1";
d98a3884 4362 else
e5bf7a7a 4363 return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
bcaec148 4364}
4365 [(set (attr "neon_type")
4366 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4367 (const_string "neon_vld1_1_2_regs")
4368 (const_string "neon_vld1_vld2_lane")))]
4369)
d98a3884 4370
4371(define_insn "neon_vld1_dup<mode>"
4372 [(set (match_operand:VDX 0 "s_register_operand" "=w")
f983358a 4373 (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
d98a3884 4374 "TARGET_NEON"
4375{
4376 if (GET_MODE_NUNITS (<MODE>mode) > 1)
e5bf7a7a 4377 return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
d98a3884 4378 else
e5bf7a7a 4379 return "vld1.<V_sz_elem>\t%h0, %A1";
bcaec148 4380}
4381 [(set (attr "neon_type")
4382 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4383 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4384 (const_string "neon_vld1_1_2_regs")))]
4385)
d98a3884 4386
4387(define_insn "neon_vld1_dup<mode>"
f983358a 4388 [(set (match_operand:VQ 0 "s_register_operand" "=w")
4389 (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
d98a3884 4390 "TARGET_NEON"
4391{
f983358a 4392 return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
bcaec148 4393}
f983358a 4394 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4395)
4396
4397(define_insn_and_split "neon_vld1_dupv2di"
4398 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4399 (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4400 "TARGET_NEON"
4401 "#"
4402 "&& reload_completed"
4403 [(const_int 0)]
4404 {
4405 rtx tmprtx = gen_lowpart (DImode, operands[0]);
4406 emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4407 emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4408 DONE;
4409 }
4410 [(set_attr "length" "8")
4411 (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
bcaec148 4412)
d98a3884 4413
672b3f5b 4414(define_expand "vec_store_lanes<mode><mode>"
4415 [(set (match_operand:VDQX 0 "neon_struct_operand")
4416 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4417 UNSPEC_VST1))]
4418 "TARGET_NEON")
4419
d98a3884 4420(define_insn "neon_vst1<mode>"
e5bf7a7a 4421 [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
d98a3884 4422 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4423 UNSPEC_VST1))]
4424 "TARGET_NEON"
e5bf7a7a 4425 "vst1.<V_sz_elem>\t%h1, %A0"
bcaec148 4426 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
d98a3884 4427
4428(define_insn "neon_vst1_lane<mode>"
e5bf7a7a 4429 [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
d98a3884 4430 (vec_select:<V_elem>
4431 (match_operand:VDX 1 "s_register_operand" "w")
4432 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4433 "TARGET_NEON"
4434{
4435 HOST_WIDE_INT lane = INTVAL (operands[2]);
4436 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4437 if (lane < 0 || lane >= max)
4438 error ("lane out of range");
4439 if (max == 1)
e5bf7a7a 4440 return "vst1.<V_sz_elem>\t{%P1}, %A0";
d98a3884 4441 else
e5bf7a7a 4442 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
bcaec148 4443}
4444 [(set (attr "neon_type")
4445 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4446 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4447 (const_string "neon_vst1_vst2_lane")))])
d98a3884 4448
4449(define_insn "neon_vst1_lane<mode>"
e5bf7a7a 4450 [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
d98a3884 4451 (vec_select:<V_elem>
4452 (match_operand:VQX 1 "s_register_operand" "w")
4453 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4454 "TARGET_NEON"
4455{
4456 HOST_WIDE_INT lane = INTVAL (operands[2]);
4457 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4458 int regno = REGNO (operands[1]);
4459 if (lane < 0 || lane >= max)
4460 error ("lane out of range");
4461 else if (lane >= max / 2)
4462 {
4463 lane -= max / 2;
4464 regno += 2;
4465 operands[2] = GEN_INT (lane);
4466 }
4467 operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4468 if (max == 2)
e5bf7a7a 4469 return "vst1.<V_sz_elem>\t{%P1}, %A0";
d98a3884 4470 else
e5bf7a7a 4471 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
bcaec148 4472}
4473 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4474)
d98a3884 4475
672b3f5b 4476(define_expand "vec_load_lanesti<mode>"
4477 [(set (match_operand:TI 0 "s_register_operand")
4478 (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4479 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4480 UNSPEC_VLD2))]
4481 "TARGET_NEON")
4482
d98a3884 4483(define_insn "neon_vld2<mode>"
4484 [(set (match_operand:TI 0 "s_register_operand" "=w")
e5bf7a7a 4485 (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
d98a3884 4486 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4487 UNSPEC_VLD2))]
4488 "TARGET_NEON"
4489{
4490 if (<V_sz_elem> == 64)
e5bf7a7a 4491 return "vld1.64\t%h0, %A1";
d98a3884 4492 else
e5bf7a7a 4493 return "vld2.<V_sz_elem>\t%h0, %A1";
bcaec148 4494}
4495 [(set (attr "neon_type")
4496 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4497 (const_string "neon_vld1_1_2_regs")
4498 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4499)
d98a3884 4500
672b3f5b 4501(define_expand "vec_load_lanesoi<mode>"
4502 [(set (match_operand:OI 0 "s_register_operand")
4503 (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4504 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4505 UNSPEC_VLD2))]
4506 "TARGET_NEON")
4507
d98a3884 4508(define_insn "neon_vld2<mode>"
4509 [(set (match_operand:OI 0 "s_register_operand" "=w")
e5bf7a7a 4510 (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
d98a3884 4511 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4512 UNSPEC_VLD2))]
4513 "TARGET_NEON"
e5bf7a7a 4514 "vld2.<V_sz_elem>\t%h0, %A1"
bcaec148 4515 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
d98a3884 4516
4517(define_insn "neon_vld2_lane<mode>"
4518 [(set (match_operand:TI 0 "s_register_operand" "=w")
e5bf7a7a 4519 (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
d98a3884 4520 (match_operand:TI 2 "s_register_operand" "0")
4521 (match_operand:SI 3 "immediate_operand" "i")
4522 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4523 UNSPEC_VLD2_LANE))]
4524 "TARGET_NEON"
4525{
4526 HOST_WIDE_INT lane = INTVAL (operands[3]);
4527 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4528 int regno = REGNO (operands[0]);
4529 rtx ops[4];
4530 if (lane < 0 || lane >= max)
4531 error ("lane out of range");
4532 ops[0] = gen_rtx_REG (DImode, regno);
4533 ops[1] = gen_rtx_REG (DImode, regno + 2);
4534 ops[2] = operands[1];
4535 ops[3] = operands[3];
e5bf7a7a 4536 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
d98a3884 4537 return "";
bcaec148 4538}
4539 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4540)
d98a3884 4541
4542(define_insn "neon_vld2_lane<mode>"
4543 [(set (match_operand:OI 0 "s_register_operand" "=w")
e5bf7a7a 4544 (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
d98a3884 4545 (match_operand:OI 2 "s_register_operand" "0")
4546 (match_operand:SI 3 "immediate_operand" "i")
4547 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4548 UNSPEC_VLD2_LANE))]
4549 "TARGET_NEON"
4550{
4551 HOST_WIDE_INT lane = INTVAL (operands[3]);
4552 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4553 int regno = REGNO (operands[0]);
4554 rtx ops[4];
4555 if (lane < 0 || lane >= max)
4556 error ("lane out of range");
4557 else if (lane >= max / 2)
4558 {
4559 lane -= max / 2;
4560 regno += 2;
4561 }
4562 ops[0] = gen_rtx_REG (DImode, regno);
4563 ops[1] = gen_rtx_REG (DImode, regno + 4);
4564 ops[2] = operands[1];
4565 ops[3] = GEN_INT (lane);
e5bf7a7a 4566 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
d98a3884 4567 return "";
bcaec148 4568}
4569 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4570)
d98a3884 4571
4572(define_insn "neon_vld2_dup<mode>"
4573 [(set (match_operand:TI 0 "s_register_operand" "=w")
e5bf7a7a 4574 (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
d98a3884 4575 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4576 UNSPEC_VLD2_DUP))]
4577 "TARGET_NEON"
4578{
4579 if (GET_MODE_NUNITS (<MODE>mode) > 1)
e5bf7a7a 4580 return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
d98a3884 4581 else
e5bf7a7a 4582 return "vld1.<V_sz_elem>\t%h0, %A1";
bcaec148 4583}
4584 [(set (attr "neon_type")
4585 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4586 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4587 (const_string "neon_vld1_1_2_regs")))]
4588)
d98a3884 4589
672b3f5b 4590(define_expand "vec_store_lanesti<mode>"
4591 [(set (match_operand:TI 0 "neon_struct_operand")
4592 (unspec:TI [(match_operand:TI 1 "s_register_operand")
4593 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4594 UNSPEC_VST2))]
4595 "TARGET_NEON")
4596
d98a3884 4597(define_insn "neon_vst2<mode>"
e5bf7a7a 4598 [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
d98a3884 4599 (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4600 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4601 UNSPEC_VST2))]
4602 "TARGET_NEON"
4603{
4604 if (<V_sz_elem> == 64)
e5bf7a7a 4605 return "vst1.64\t%h1, %A0";
d98a3884 4606 else
e5bf7a7a 4607 return "vst2.<V_sz_elem>\t%h1, %A0";
bcaec148 4608}
4609 [(set (attr "neon_type")
4610 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4611 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4612 (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4613)
d98a3884 4614
672b3f5b 4615(define_expand "vec_store_lanesoi<mode>"
4616 [(set (match_operand:OI 0 "neon_struct_operand")
4617 (unspec:OI [(match_operand:OI 1 "s_register_operand")
4618 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4619 UNSPEC_VST2))]
4620 "TARGET_NEON")
4621
d98a3884 4622(define_insn "neon_vst2<mode>"
e5bf7a7a 4623 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
d98a3884 4624 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4625 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4626 UNSPEC_VST2))]
4627 "TARGET_NEON"
e5bf7a7a 4628 "vst2.<V_sz_elem>\t%h1, %A0"
bcaec148 4629 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4630)
d98a3884 4631
4632(define_insn "neon_vst2_lane<mode>"
e5bf7a7a 4633 [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
d98a3884 4634 (unspec:<V_two_elem>
4635 [(match_operand:TI 1 "s_register_operand" "w")
4636 (match_operand:SI 2 "immediate_operand" "i")
4637 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4638 UNSPEC_VST2_LANE))]
4639 "TARGET_NEON"
4640{
4641 HOST_WIDE_INT lane = INTVAL (operands[2]);
4642 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4643 int regno = REGNO (operands[1]);
4644 rtx ops[4];
4645 if (lane < 0 || lane >= max)
4646 error ("lane out of range");
4647 ops[0] = operands[0];
4648 ops[1] = gen_rtx_REG (DImode, regno);
4649 ops[2] = gen_rtx_REG (DImode, regno + 2);
4650 ops[3] = operands[2];
e5bf7a7a 4651 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
d98a3884 4652 return "";
bcaec148 4653}
4654 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4655)
d98a3884 4656
4657(define_insn "neon_vst2_lane<mode>"
e5bf7a7a 4658 [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
d98a3884 4659 (unspec:<V_two_elem>
4660 [(match_operand:OI 1 "s_register_operand" "w")
4661 (match_operand:SI 2 "immediate_operand" "i")
4662 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4663 UNSPEC_VST2_LANE))]
4664 "TARGET_NEON"
4665{
4666 HOST_WIDE_INT lane = INTVAL (operands[2]);
4667 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4668 int regno = REGNO (operands[1]);
4669 rtx ops[4];
4670 if (lane < 0 || lane >= max)
4671 error ("lane out of range");
4672 else if (lane >= max / 2)
4673 {
4674 lane -= max / 2;
4675 regno += 2;
4676 }
4677 ops[0] = operands[0];
4678 ops[1] = gen_rtx_REG (DImode, regno);
4679 ops[2] = gen_rtx_REG (DImode, regno + 4);
4680 ops[3] = GEN_INT (lane);
e5bf7a7a 4681 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
d98a3884 4682 return "";
bcaec148 4683}
4684 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4685)
d98a3884 4686
672b3f5b 4687(define_expand "vec_load_lanesei<mode>"
4688 [(set (match_operand:EI 0 "s_register_operand")
4689 (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4690 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4691 UNSPEC_VLD3))]
4692 "TARGET_NEON")
4693
d98a3884 4694(define_insn "neon_vld3<mode>"
4695 [(set (match_operand:EI 0 "s_register_operand" "=w")
e5bf7a7a 4696 (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
d98a3884 4697 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4698 UNSPEC_VLD3))]
4699 "TARGET_NEON"
4700{
4701 if (<V_sz_elem> == 64)
e5bf7a7a 4702 return "vld1.64\t%h0, %A1";
d98a3884 4703 else
e5bf7a7a 4704 return "vld3.<V_sz_elem>\t%h0, %A1";
bcaec148 4705}
4706 [(set (attr "neon_type")
4707 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4708 (const_string "neon_vld1_1_2_regs")
4709 (const_string "neon_vld3_vld4")))]
4710)
d98a3884 4711
672b3f5b 4712(define_expand "vec_load_lanesci<mode>"
4713 [(match_operand:CI 0 "s_register_operand")
4714 (match_operand:CI 1 "neon_struct_operand")
4715 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4716 "TARGET_NEON"
4717{
4718 emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4719 DONE;
4720})
4721
d98a3884 4722(define_expand "neon_vld3<mode>"
e5bf7a7a 4723 [(match_operand:CI 0 "s_register_operand")
4724 (match_operand:CI 1 "neon_struct_operand")
d98a3884 4725 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4726 "TARGET_NEON"
4727{
e5bf7a7a 4728 rtx mem;
4729
4730 mem = adjust_address (operands[1], EImode, 0);
4731 emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4732 mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4733 emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
d98a3884 4734 DONE;
4735})
4736
4737(define_insn "neon_vld3qa<mode>"
4738 [(set (match_operand:CI 0 "s_register_operand" "=w")
e5bf7a7a 4739 (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
d98a3884 4740 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 4741 UNSPEC_VLD3A))]
d98a3884 4742 "TARGET_NEON"
4743{
4744 int regno = REGNO (operands[0]);
4745 rtx ops[4];
4746 ops[0] = gen_rtx_REG (DImode, regno);
4747 ops[1] = gen_rtx_REG (DImode, regno + 4);
4748 ops[2] = gen_rtx_REG (DImode, regno + 8);
92d6c32b 4749 ops[3] = operands[1];
e5bf7a7a 4750 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
d98a3884 4751 return "";
bcaec148 4752}
4753 [(set_attr "neon_type" "neon_vld3_vld4")]
4754)
d98a3884 4755
4756(define_insn "neon_vld3qb<mode>"
4757 [(set (match_operand:CI 0 "s_register_operand" "=w")
e5bf7a7a 4758 (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4759 (match_operand:CI 2 "s_register_operand" "0")
d98a3884 4760 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 4761 UNSPEC_VLD3B))]
d98a3884 4762 "TARGET_NEON"
4763{
4764 int regno = REGNO (operands[0]);
4765 rtx ops[4];
4766 ops[0] = gen_rtx_REG (DImode, regno + 2);
4767 ops[1] = gen_rtx_REG (DImode, regno + 6);
4768 ops[2] = gen_rtx_REG (DImode, regno + 10);
e5bf7a7a 4769 ops[3] = operands[1];
4770 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
d98a3884 4771 return "";
bcaec148 4772}
4773 [(set_attr "neon_type" "neon_vld3_vld4")]
4774)
d98a3884 4775
4776(define_insn "neon_vld3_lane<mode>"
4777 [(set (match_operand:EI 0 "s_register_operand" "=w")
e5bf7a7a 4778 (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
d98a3884 4779 (match_operand:EI 2 "s_register_operand" "0")
4780 (match_operand:SI 3 "immediate_operand" "i")
4781 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4782 UNSPEC_VLD3_LANE))]
4783 "TARGET_NEON"
4784{
4785 HOST_WIDE_INT lane = INTVAL (operands[3]);
4786 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4787 int regno = REGNO (operands[0]);
4788 rtx ops[5];
4789 if (lane < 0 || lane >= max)
4790 error ("lane out of range");
4791 ops[0] = gen_rtx_REG (DImode, regno);
4792 ops[1] = gen_rtx_REG (DImode, regno + 2);
4793 ops[2] = gen_rtx_REG (DImode, regno + 4);
4794 ops[3] = operands[1];
4795 ops[4] = operands[3];
0b66dd3d 4796 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
d98a3884 4797 ops);
4798 return "";
bcaec148 4799}
4800 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4801)
d98a3884 4802
4803(define_insn "neon_vld3_lane<mode>"
4804 [(set (match_operand:CI 0 "s_register_operand" "=w")
e5bf7a7a 4805 (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
d98a3884 4806 (match_operand:CI 2 "s_register_operand" "0")
4807 (match_operand:SI 3 "immediate_operand" "i")
4808 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4809 UNSPEC_VLD3_LANE))]
4810 "TARGET_NEON"
4811{
4812 HOST_WIDE_INT lane = INTVAL (operands[3]);
4813 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4814 int regno = REGNO (operands[0]);
4815 rtx ops[5];
4816 if (lane < 0 || lane >= max)
4817 error ("lane out of range");
4818 else if (lane >= max / 2)
4819 {
4820 lane -= max / 2;
4821 regno += 2;
4822 }
4823 ops[0] = gen_rtx_REG (DImode, regno);
4824 ops[1] = gen_rtx_REG (DImode, regno + 4);
4825 ops[2] = gen_rtx_REG (DImode, regno + 8);
4826 ops[3] = operands[1];
4827 ops[4] = GEN_INT (lane);
0b66dd3d 4828 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
d98a3884 4829 ops);
4830 return "";
bcaec148 4831}
4832 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4833)
d98a3884 4834
4835(define_insn "neon_vld3_dup<mode>"
4836 [(set (match_operand:EI 0 "s_register_operand" "=w")
e5bf7a7a 4837 (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
d98a3884 4838 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4839 UNSPEC_VLD3_DUP))]
4840 "TARGET_NEON"
4841{
4842 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4843 {
4844 int regno = REGNO (operands[0]);
4845 rtx ops[4];
4846 ops[0] = gen_rtx_REG (DImode, regno);
4847 ops[1] = gen_rtx_REG (DImode, regno + 2);
4848 ops[2] = gen_rtx_REG (DImode, regno + 4);
4849 ops[3] = operands[1];
0b66dd3d 4850 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
d98a3884 4851 return "";
4852 }
4853 else
e5bf7a7a 4854 return "vld1.<V_sz_elem>\t%h0, %A1";
bcaec148 4855}
4856 [(set (attr "neon_type")
4857 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4858 (const_string "neon_vld3_vld4_all_lanes")
4859 (const_string "neon_vld1_1_2_regs")))])
d98a3884 4860
672b3f5b 4861(define_expand "vec_store_lanesei<mode>"
4862 [(set (match_operand:EI 0 "neon_struct_operand")
4863 (unspec:EI [(match_operand:EI 1 "s_register_operand")
4864 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4865 UNSPEC_VST3))]
4866 "TARGET_NEON")
4867
d98a3884 4868(define_insn "neon_vst3<mode>"
e5bf7a7a 4869 [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
d98a3884 4870 (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4871 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4872 UNSPEC_VST3))]
4873 "TARGET_NEON"
4874{
4875 if (<V_sz_elem> == 64)
e5bf7a7a 4876 return "vst1.64\t%h1, %A0";
d98a3884 4877 else
e5bf7a7a 4878 return "vst3.<V_sz_elem>\t%h1, %A0";
bcaec148 4879}
4880 [(set (attr "neon_type")
4881 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4882 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4883 (const_string "neon_vst2_4_regs_vst3_vst4")))])
d98a3884 4884
672b3f5b 4885(define_expand "vec_store_lanesci<mode>"
4886 [(match_operand:CI 0 "neon_struct_operand")
4887 (match_operand:CI 1 "s_register_operand")
4888 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4889 "TARGET_NEON"
4890{
4891 emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
4892 DONE;
4893})
4894
d98a3884 4895(define_expand "neon_vst3<mode>"
e5bf7a7a 4896 [(match_operand:CI 0 "neon_struct_operand")
4897 (match_operand:CI 1 "s_register_operand")
d98a3884 4898 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4899 "TARGET_NEON"
4900{
e5bf7a7a 4901 rtx mem;
4902
4903 mem = adjust_address (operands[0], EImode, 0);
4904 emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
4905 mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4906 emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
d98a3884 4907 DONE;
4908})
4909
4910(define_insn "neon_vst3qa<mode>"
e5bf7a7a 4911 [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4912 (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
d98a3884 4913 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 4914 UNSPEC_VST3A))]
d98a3884 4915 "TARGET_NEON"
4916{
e5bf7a7a 4917 int regno = REGNO (operands[1]);
d98a3884 4918 rtx ops[4];
4919 ops[0] = operands[0];
4920 ops[1] = gen_rtx_REG (DImode, regno);
4921 ops[2] = gen_rtx_REG (DImode, regno + 4);
4922 ops[3] = gen_rtx_REG (DImode, regno + 8);
e5bf7a7a 4923 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
d98a3884 4924 return "";
bcaec148 4925}
4926 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4927)
d98a3884 4928
4929(define_insn "neon_vst3qb<mode>"
e5bf7a7a 4930 [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4931 (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
d98a3884 4932 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 4933 UNSPEC_VST3B))]
d98a3884 4934 "TARGET_NEON"
4935{
e5bf7a7a 4936 int regno = REGNO (operands[1]);
d98a3884 4937 rtx ops[4];
4938 ops[0] = operands[0];
4939 ops[1] = gen_rtx_REG (DImode, regno + 2);
4940 ops[2] = gen_rtx_REG (DImode, regno + 6);
4941 ops[3] = gen_rtx_REG (DImode, regno + 10);
e5bf7a7a 4942 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
d98a3884 4943 return "";
bcaec148 4944}
4945 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4946)
d98a3884 4947
4948(define_insn "neon_vst3_lane<mode>"
e5bf7a7a 4949 [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
d98a3884 4950 (unspec:<V_three_elem>
4951 [(match_operand:EI 1 "s_register_operand" "w")
4952 (match_operand:SI 2 "immediate_operand" "i")
4953 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4954 UNSPEC_VST3_LANE))]
4955 "TARGET_NEON"
4956{
4957 HOST_WIDE_INT lane = INTVAL (operands[2]);
4958 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4959 int regno = REGNO (operands[1]);
4960 rtx ops[5];
4961 if (lane < 0 || lane >= max)
4962 error ("lane out of range");
4963 ops[0] = operands[0];
4964 ops[1] = gen_rtx_REG (DImode, regno);
4965 ops[2] = gen_rtx_REG (DImode, regno + 2);
4966 ops[3] = gen_rtx_REG (DImode, regno + 4);
4967 ops[4] = operands[2];
0b66dd3d 4968 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
d98a3884 4969 ops);
4970 return "";
bcaec148 4971}
4972 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
4973)
d98a3884 4974
4975(define_insn "neon_vst3_lane<mode>"
e5bf7a7a 4976 [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
d98a3884 4977 (unspec:<V_three_elem>
4978 [(match_operand:CI 1 "s_register_operand" "w")
4979 (match_operand:SI 2 "immediate_operand" "i")
4980 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4981 UNSPEC_VST3_LANE))]
4982 "TARGET_NEON"
4983{
4984 HOST_WIDE_INT lane = INTVAL (operands[2]);
4985 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4986 int regno = REGNO (operands[1]);
4987 rtx ops[5];
4988 if (lane < 0 || lane >= max)
4989 error ("lane out of range");
4990 else if (lane >= max / 2)
4991 {
4992 lane -= max / 2;
4993 regno += 2;
4994 }
4995 ops[0] = operands[0];
4996 ops[1] = gen_rtx_REG (DImode, regno);
4997 ops[2] = gen_rtx_REG (DImode, regno + 4);
4998 ops[3] = gen_rtx_REG (DImode, regno + 8);
4999 ops[4] = GEN_INT (lane);
0b66dd3d 5000 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
d98a3884 5001 ops);
5002 return "";
bcaec148 5003}
5004[(set_attr "neon_type" "neon_vst3_vst4_lane")])
d98a3884 5005
672b3f5b 5006(define_expand "vec_load_lanesoi<mode>"
5007 [(set (match_operand:OI 0 "s_register_operand")
5008 (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5009 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5010 UNSPEC_VLD4))]
5011 "TARGET_NEON")
5012
d98a3884 5013(define_insn "neon_vld4<mode>"
5014 [(set (match_operand:OI 0 "s_register_operand" "=w")
e5bf7a7a 5015 (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
d98a3884 5016 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5017 UNSPEC_VLD4))]
5018 "TARGET_NEON"
5019{
5020 if (<V_sz_elem> == 64)
e5bf7a7a 5021 return "vld1.64\t%h0, %A1";
d98a3884 5022 else
e5bf7a7a 5023 return "vld4.<V_sz_elem>\t%h0, %A1";
bcaec148 5024}
5025 [(set (attr "neon_type")
5026 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5027 (const_string "neon_vld1_1_2_regs")
5028 (const_string "neon_vld3_vld4")))]
5029)
d98a3884 5030
672b3f5b 5031(define_expand "vec_load_lanesxi<mode>"
5032 [(match_operand:XI 0 "s_register_operand")
5033 (match_operand:XI 1 "neon_struct_operand")
5034 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5035 "TARGET_NEON"
5036{
5037 emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5038 DONE;
5039})
5040
d98a3884 5041(define_expand "neon_vld4<mode>"
e5bf7a7a 5042 [(match_operand:XI 0 "s_register_operand")
5043 (match_operand:XI 1 "neon_struct_operand")
d98a3884 5044 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5045 "TARGET_NEON"
5046{
e5bf7a7a 5047 rtx mem;
5048
5049 mem = adjust_address (operands[1], OImode, 0);
5050 emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5051 mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5052 emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
d98a3884 5053 DONE;
5054})
5055
5056(define_insn "neon_vld4qa<mode>"
5057 [(set (match_operand:XI 0 "s_register_operand" "=w")
e5bf7a7a 5058 (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
d98a3884 5059 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 5060 UNSPEC_VLD4A))]
d98a3884 5061 "TARGET_NEON"
5062{
5063 int regno = REGNO (operands[0]);
5064 rtx ops[5];
5065 ops[0] = gen_rtx_REG (DImode, regno);
5066 ops[1] = gen_rtx_REG (DImode, regno + 4);
5067 ops[2] = gen_rtx_REG (DImode, regno + 8);
5068 ops[3] = gen_rtx_REG (DImode, regno + 12);
92d6c32b 5069 ops[4] = operands[1];
e5bf7a7a 5070 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
d98a3884 5071 return "";
bcaec148 5072}
5073 [(set_attr "neon_type" "neon_vld3_vld4")]
5074)
d98a3884 5075
5076(define_insn "neon_vld4qb<mode>"
5077 [(set (match_operand:XI 0 "s_register_operand" "=w")
e5bf7a7a 5078 (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5079 (match_operand:XI 2 "s_register_operand" "0")
d98a3884 5080 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 5081 UNSPEC_VLD4B))]
d98a3884 5082 "TARGET_NEON"
5083{
5084 int regno = REGNO (operands[0]);
5085 rtx ops[5];
5086 ops[0] = gen_rtx_REG (DImode, regno + 2);
5087 ops[1] = gen_rtx_REG (DImode, regno + 6);
5088 ops[2] = gen_rtx_REG (DImode, regno + 10);
5089 ops[3] = gen_rtx_REG (DImode, regno + 14);
e5bf7a7a 5090 ops[4] = operands[1];
5091 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
d98a3884 5092 return "";
bcaec148 5093}
5094 [(set_attr "neon_type" "neon_vld3_vld4")]
5095)
d98a3884 5096
5097(define_insn "neon_vld4_lane<mode>"
5098 [(set (match_operand:OI 0 "s_register_operand" "=w")
e5bf7a7a 5099 (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
d98a3884 5100 (match_operand:OI 2 "s_register_operand" "0")
5101 (match_operand:SI 3 "immediate_operand" "i")
5102 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5103 UNSPEC_VLD4_LANE))]
5104 "TARGET_NEON"
5105{
5106 HOST_WIDE_INT lane = INTVAL (operands[3]);
5107 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5108 int regno = REGNO (operands[0]);
5109 rtx ops[6];
5110 if (lane < 0 || lane >= max)
5111 error ("lane out of range");
5112 ops[0] = gen_rtx_REG (DImode, regno);
5113 ops[1] = gen_rtx_REG (DImode, regno + 2);
5114 ops[2] = gen_rtx_REG (DImode, regno + 4);
5115 ops[3] = gen_rtx_REG (DImode, regno + 6);
5116 ops[4] = operands[1];
5117 ops[5] = operands[3];
e5bf7a7a 5118 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
d98a3884 5119 ops);
5120 return "";
bcaec148 5121}
5122 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5123)
d98a3884 5124
5125(define_insn "neon_vld4_lane<mode>"
5126 [(set (match_operand:XI 0 "s_register_operand" "=w")
e5bf7a7a 5127 (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
d98a3884 5128 (match_operand:XI 2 "s_register_operand" "0")
5129 (match_operand:SI 3 "immediate_operand" "i")
5130 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5131 UNSPEC_VLD4_LANE))]
5132 "TARGET_NEON"
5133{
5134 HOST_WIDE_INT lane = INTVAL (operands[3]);
5135 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5136 int regno = REGNO (operands[0]);
5137 rtx ops[6];
5138 if (lane < 0 || lane >= max)
5139 error ("lane out of range");
5140 else if (lane >= max / 2)
5141 {
5142 lane -= max / 2;
5143 regno += 2;
5144 }
5145 ops[0] = gen_rtx_REG (DImode, regno);
5146 ops[1] = gen_rtx_REG (DImode, regno + 4);
5147 ops[2] = gen_rtx_REG (DImode, regno + 8);
5148 ops[3] = gen_rtx_REG (DImode, regno + 12);
5149 ops[4] = operands[1];
5150 ops[5] = GEN_INT (lane);
e5bf7a7a 5151 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
d98a3884 5152 ops);
5153 return "";
bcaec148 5154}
5155 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5156)
d98a3884 5157
5158(define_insn "neon_vld4_dup<mode>"
5159 [(set (match_operand:OI 0 "s_register_operand" "=w")
e5bf7a7a 5160 (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
d98a3884 5161 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5162 UNSPEC_VLD4_DUP))]
5163 "TARGET_NEON"
5164{
5165 if (GET_MODE_NUNITS (<MODE>mode) > 1)
5166 {
5167 int regno = REGNO (operands[0]);
5168 rtx ops[5];
5169 ops[0] = gen_rtx_REG (DImode, regno);
5170 ops[1] = gen_rtx_REG (DImode, regno + 2);
5171 ops[2] = gen_rtx_REG (DImode, regno + 4);
5172 ops[3] = gen_rtx_REG (DImode, regno + 6);
5173 ops[4] = operands[1];
e5bf7a7a 5174 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
d98a3884 5175 ops);
5176 return "";
5177 }
5178 else
e5bf7a7a 5179 return "vld1.<V_sz_elem>\t%h0, %A1";
bcaec148 5180}
5181 [(set (attr "neon_type")
5182 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5183 (const_string "neon_vld3_vld4_all_lanes")
5184 (const_string "neon_vld1_1_2_regs")))]
5185)
d98a3884 5186
672b3f5b 5187(define_expand "vec_store_lanesoi<mode>"
5188 [(set (match_operand:OI 0 "neon_struct_operand")
5189 (unspec:OI [(match_operand:OI 1 "s_register_operand")
5190 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5191 UNSPEC_VST4))]
5192 "TARGET_NEON")
5193
d98a3884 5194(define_insn "neon_vst4<mode>"
e5bf7a7a 5195 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
d98a3884 5196 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5197 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5198 UNSPEC_VST4))]
5199 "TARGET_NEON"
5200{
5201 if (<V_sz_elem> == 64)
e5bf7a7a 5202 return "vst1.64\t%h1, %A0";
d98a3884 5203 else
e5bf7a7a 5204 return "vst4.<V_sz_elem>\t%h1, %A0";
bcaec148 5205}
5206 [(set (attr "neon_type")
5207 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5208 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5209 (const_string "neon_vst2_4_regs_vst3_vst4")))]
5210)
d98a3884 5211
672b3f5b 5212(define_expand "vec_store_lanesxi<mode>"
5213 [(match_operand:XI 0 "neon_struct_operand")
5214 (match_operand:XI 1 "s_register_operand")
5215 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5216 "TARGET_NEON"
5217{
5218 emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5219 DONE;
5220})
5221
d98a3884 5222(define_expand "neon_vst4<mode>"
e5bf7a7a 5223 [(match_operand:XI 0 "neon_struct_operand")
5224 (match_operand:XI 1 "s_register_operand")
d98a3884 5225 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5226 "TARGET_NEON"
5227{
e5bf7a7a 5228 rtx mem;
5229
5230 mem = adjust_address (operands[0], OImode, 0);
5231 emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5232 mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5233 emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
d98a3884 5234 DONE;
5235})
5236
5237(define_insn "neon_vst4qa<mode>"
e5bf7a7a 5238 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5239 (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
d98a3884 5240 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 5241 UNSPEC_VST4A))]
d98a3884 5242 "TARGET_NEON"
5243{
e5bf7a7a 5244 int regno = REGNO (operands[1]);
d98a3884 5245 rtx ops[5];
5246 ops[0] = operands[0];
5247 ops[1] = gen_rtx_REG (DImode, regno);
5248 ops[2] = gen_rtx_REG (DImode, regno + 4);
5249 ops[3] = gen_rtx_REG (DImode, regno + 8);
5250 ops[4] = gen_rtx_REG (DImode, regno + 12);
e5bf7a7a 5251 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
d98a3884 5252 return "";
bcaec148 5253}
5254 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5255)
d98a3884 5256
5257(define_insn "neon_vst4qb<mode>"
e5bf7a7a 5258 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5259 (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
d98a3884 5260 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
e5bf7a7a 5261 UNSPEC_VST4B))]
d98a3884 5262 "TARGET_NEON"
5263{
e5bf7a7a 5264 int regno = REGNO (operands[1]);
d98a3884 5265 rtx ops[5];
5266 ops[0] = operands[0];
5267 ops[1] = gen_rtx_REG (DImode, regno + 2);
5268 ops[2] = gen_rtx_REG (DImode, regno + 6);
5269 ops[3] = gen_rtx_REG (DImode, regno + 10);
5270 ops[4] = gen_rtx_REG (DImode, regno + 14);
e5bf7a7a 5271 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
d98a3884 5272 return "";
bcaec148 5273}
5274 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5275)
d98a3884 5276
5277(define_insn "neon_vst4_lane<mode>"
e5bf7a7a 5278 [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
d98a3884 5279 (unspec:<V_four_elem>
5280 [(match_operand:OI 1 "s_register_operand" "w")
5281 (match_operand:SI 2 "immediate_operand" "i")
5282 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5283 UNSPEC_VST4_LANE))]
5284 "TARGET_NEON"
5285{
5286 HOST_WIDE_INT lane = INTVAL (operands[2]);
5287 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5288 int regno = REGNO (operands[1]);
5289 rtx ops[6];
5290 if (lane < 0 || lane >= max)
5291 error ("lane out of range");
5292 ops[0] = operands[0];
5293 ops[1] = gen_rtx_REG (DImode, regno);
5294 ops[2] = gen_rtx_REG (DImode, regno + 2);
5295 ops[3] = gen_rtx_REG (DImode, regno + 4);
5296 ops[4] = gen_rtx_REG (DImode, regno + 6);
5297 ops[5] = operands[2];
e5bf7a7a 5298 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
d98a3884 5299 ops);
5300 return "";
bcaec148 5301}
5302 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5303)
d98a3884 5304
5305(define_insn "neon_vst4_lane<mode>"
e5bf7a7a 5306 [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
d98a3884 5307 (unspec:<V_four_elem>
5308 [(match_operand:XI 1 "s_register_operand" "w")
5309 (match_operand:SI 2 "immediate_operand" "i")
5310 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5311 UNSPEC_VST4_LANE))]
5312 "TARGET_NEON"
5313{
5314 HOST_WIDE_INT lane = INTVAL (operands[2]);
5315 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5316 int regno = REGNO (operands[1]);
5317 rtx ops[6];
5318 if (lane < 0 || lane >= max)
5319 error ("lane out of range");
5320 else if (lane >= max / 2)
5321 {
5322 lane -= max / 2;
5323 regno += 2;
5324 }
5325 ops[0] = operands[0];
5326 ops[1] = gen_rtx_REG (DImode, regno);
5327 ops[2] = gen_rtx_REG (DImode, regno + 4);
5328 ops[3] = gen_rtx_REG (DImode, regno + 8);
5329 ops[4] = gen_rtx_REG (DImode, regno + 12);
5330 ops[5] = GEN_INT (lane);
e5bf7a7a 5331 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
d98a3884 5332 ops);
5333 return "";
bcaec148 5334}
5335 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5336)
d98a3884 5337
5338(define_expand "neon_vand<mode>"
5339 [(match_operand:VDQX 0 "s_register_operand" "")
5340 (match_operand:VDQX 1 "s_register_operand" "")
5341 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5342 (match_operand:SI 3 "immediate_operand" "")]
5343 "TARGET_NEON"
5344{
5345 emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5346 DONE;
5347})
5348
5349(define_expand "neon_vorr<mode>"
5350 [(match_operand:VDQX 0 "s_register_operand" "")
5351 (match_operand:VDQX 1 "s_register_operand" "")
5352 (match_operand:VDQX 2 "neon_logic_op2" "")
5353 (match_operand:SI 3 "immediate_operand" "")]
5354 "TARGET_NEON"
5355{
5356 emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5357 DONE;
5358})
5359
5360(define_expand "neon_veor<mode>"
5361 [(match_operand:VDQX 0 "s_register_operand" "")
5362 (match_operand:VDQX 1 "s_register_operand" "")
5363 (match_operand:VDQX 2 "s_register_operand" "")
5364 (match_operand:SI 3 "immediate_operand" "")]
5365 "TARGET_NEON"
5366{
5367 emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5368 DONE;
5369})
5370
5371(define_expand "neon_vbic<mode>"
5372 [(match_operand:VDQX 0 "s_register_operand" "")
5373 (match_operand:VDQX 1 "s_register_operand" "")
5374 (match_operand:VDQX 2 "neon_logic_op2" "")
5375 (match_operand:SI 3 "immediate_operand" "")]
5376 "TARGET_NEON"
5377{
5378 emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5379 DONE;
5380})
5381
5382(define_expand "neon_vorn<mode>"
5383 [(match_operand:VDQX 0 "s_register_operand" "")
5384 (match_operand:VDQX 1 "s_register_operand" "")
5385 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5386 (match_operand:SI 3 "immediate_operand" "")]
5387 "TARGET_NEON"
5388{
5389 emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5390 DONE;
5391})
6e4376d7 5392
5393(define_insn "neon_vec_unpack<US>_lo_<mode>"
5394 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5395 (SE:<V_unpack> (vec_select:<V_HALF>
5396 (match_operand:VU 1 "register_operand" "w")
5397 (match_operand:VU 2 "vect_par_constant_low" ""))))]
b46a36c7 5398 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5399 "vmovl.<US><V_sz_elem> %q0, %e1"
5400 [(set_attr "neon_type" "neon_shift_1")]
5401)
5402
5403(define_insn "neon_vec_unpack<US>_hi_<mode>"
5404 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5405 (SE:<V_unpack> (vec_select:<V_HALF>
5406 (match_operand:VU 1 "register_operand" "w")
5407 (match_operand:VU 2 "vect_par_constant_high" ""))))]
b46a36c7 5408 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5409 "vmovl.<US><V_sz_elem> %q0, %f1"
5410 [(set_attr "neon_type" "neon_shift_1")]
5411)
5412
5413(define_expand "vec_unpack<US>_hi_<mode>"
5414 [(match_operand:<V_unpack> 0 "register_operand" "")
5415 (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
b46a36c7 5416 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5417 {
5418 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5419 rtx t1;
5420 int i;
5421 for (i = 0; i < (<V_mode_nunits>/2); i++)
5422 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5423
5424 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5425 emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5426 operands[1],
5427 t1));
5428 DONE;
5429 }
5430)
5431
5432(define_expand "vec_unpack<US>_lo_<mode>"
5433 [(match_operand:<V_unpack> 0 "register_operand" "")
5434 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
b46a36c7 5435 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5436 {
5437 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5438 rtx t1;
5439 int i;
5440 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5441 RTVEC_ELT (v, i) = GEN_INT (i);
5442 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5443 emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5444 operands[1],
5445 t1));
5446 DONE;
5447 }
5448)
5449
5450(define_insn "neon_vec_<US>mult_lo_<mode>"
5451 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5452 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5453 (match_operand:VU 1 "register_operand" "w")
5454 (match_operand:VU 2 "vect_par_constant_low" "")))
5455 (SE:<V_unpack> (vec_select:<V_HALF>
5456 (match_operand:VU 3 "register_operand" "w")
5457 (match_dup 2)))))]
b46a36c7 5458 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5459 "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5460 [(set_attr "neon_type" "neon_shift_1")]
5461)
5462
5463(define_expand "vec_widen_<US>mult_lo_<mode>"
5464 [(match_operand:<V_unpack> 0 "register_operand" "")
5465 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5466 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
b46a36c7 5467 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5468 {
5469 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5470 rtx t1;
5471 int i;
5472 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5473 RTVEC_ELT (v, i) = GEN_INT (i);
5474 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5475
5476 emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5477 operands[1],
5478 t1,
5479 operands[2]));
5480 DONE;
5481 }
5482)
5483
5484(define_insn "neon_vec_<US>mult_hi_<mode>"
5485 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5486 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5487 (match_operand:VU 1 "register_operand" "w")
5488 (match_operand:VU 2 "vect_par_constant_high" "")))
5489 (SE:<V_unpack> (vec_select:<V_HALF>
5490 (match_operand:VU 3 "register_operand" "w")
5491 (match_dup 2)))))]
b46a36c7 5492 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5493 "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5494 [(set_attr "neon_type" "neon_shift_1")]
5495)
5496
5497(define_expand "vec_widen_<US>mult_hi_<mode>"
5498 [(match_operand:<V_unpack> 0 "register_operand" "")
5499 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5500 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
b46a36c7 5501 "TARGET_NEON && !BYTES_BIG_ENDIAN"
6e4376d7 5502 {
5503 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5504 rtx t1;
5505 int i;
5506 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5507 RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5508 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5509
5510 emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5511 operands[1],
5512 t1,
5513 operands[2]));
5514 DONE;
5515
5516 }
5517)
5518
6083c152 5519(define_insn "neon_vec_<US>shiftl_<mode>"
5520 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5521 (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5522 (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5523 "TARGET_NEON"
5524{
5525 return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5526}
5527 [(set_attr "neon_type" "neon_shift_1")]
5528)
5529
5530(define_expand "vec_widen_<US>shiftl_lo_<mode>"
5531 [(match_operand:<V_unpack> 0 "register_operand" "")
5532 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5533 (match_operand:SI 2 "immediate_operand" "i")]
5534 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5535 {
5536 emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5537 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5538 operands[2]));
5539 DONE;
5540 }
5541)
5542
5543(define_expand "vec_widen_<US>shiftl_hi_<mode>"
5544 [(match_operand:<V_unpack> 0 "register_operand" "")
5545 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5546 (match_operand:SI 2 "immediate_operand" "i")]
5547 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5548 {
5549 emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5550 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5551 GET_MODE_SIZE (<V_HALF>mode)),
5552 operands[2]));
5553 DONE;
5554 }
5555)
5556
6e4376d7 5557;; Vectorize for non-neon-quad case
5558(define_insn "neon_unpack<US>_<mode>"
5559 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
80d18bad 5560 (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
6e4376d7 5561 "TARGET_NEON"
80d18bad 5562 "vmovl.<US><V_sz_elem> %q0, %P1"
6e4376d7 5563 [(set_attr "neon_type" "neon_shift_1")]
5564)
5565
5566(define_expand "vec_unpack<US>_lo_<mode>"
5567 [(match_operand:<V_double_width> 0 "register_operand" "")
5568 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5569 "TARGET_NEON"
5570{
5571 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5572 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5573 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5574
5575 DONE;
5576}
5577)
5578
5579(define_expand "vec_unpack<US>_hi_<mode>"
5580 [(match_operand:<V_double_width> 0 "register_operand" "")
5581 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5582 "TARGET_NEON"
5583{
5584 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5585 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5586 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5587
5588 DONE;
5589}
5590)
5591
5592(define_insn "neon_vec_<US>mult_<mode>"
5593 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5594 (mult:<V_widen> (SE:<V_widen>
5595 (match_operand:VDI 1 "register_operand" "w"))
5596 (SE:<V_widen>
5597 (match_operand:VDI 2 "register_operand" "w"))))]
5598 "TARGET_NEON"
80d18bad 5599 "vmull.<US><V_sz_elem> %q0, %P1, %P2"
6e4376d7 5600 [(set_attr "neon_type" "neon_shift_1")]
5601)
5602
5603(define_expand "vec_widen_<US>mult_hi_<mode>"
5604 [(match_operand:<V_double_width> 0 "register_operand" "")
5605 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5606 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5607 "TARGET_NEON"
5608 {
5609 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5610 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5611 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5612
5613 DONE;
5614
5615 }
5616)
5617
5618(define_expand "vec_widen_<US>mult_lo_<mode>"
5619 [(match_operand:<V_double_width> 0 "register_operand" "")
5620 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5621 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5622 "TARGET_NEON"
5623 {
5624 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5625 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5626 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5627
5628 DONE;
5629
5630 }
5631)
a62cc977 5632
6083c152 5633(define_expand "vec_widen_<US>shiftl_hi_<mode>"
5634 [(match_operand:<V_double_width> 0 "register_operand" "")
5635 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5636 (match_operand:SI 2 "immediate_operand" "i")]
5637 "TARGET_NEON"
5638 {
5639 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5640 emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5641 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5642
5643 DONE;
5644 }
5645)
5646
5647(define_expand "vec_widen_<US>shiftl_lo_<mode>"
5648 [(match_operand:<V_double_width> 0 "register_operand" "")
5649 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5650 (match_operand:SI 2 "immediate_operand" "i")]
5651 "TARGET_NEON"
5652 {
5653 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5654 emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5655 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5656
5657 DONE;
5658 }
5659)
5660
b46a36c7 5661; FIXME: These instruction patterns can't be used safely in big-endian mode
5662; because the ordering of vector elements in Q registers is different from what
5663; the semantics of the instructions require.
5664
a62cc977 5665(define_insn "vec_pack_trunc_<mode>"
5666 [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5667 (vec_concat:<V_narrow_pack>
5668 (truncate:<V_narrow>
5669 (match_operand:VN 1 "register_operand" "w"))
5670 (truncate:<V_narrow>
5671 (match_operand:VN 2 "register_operand" "w"))))]
b46a36c7 5672 "TARGET_NEON && !BYTES_BIG_ENDIAN"
11371434 5673 "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5674 [(set_attr "neon_type" "neon_shift_1")
5675 (set_attr "length" "8")]
a62cc977 5676)
5677
5678;; For the non-quad case.
5679(define_insn "neon_vec_pack_trunc_<mode>"
5680 [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
80d18bad 5681 (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
b46a36c7 5682 "TARGET_NEON && !BYTES_BIG_ENDIAN"
80d18bad 5683 "vmovn.i<V_sz_elem>\t%P0, %q1"
a62cc977 5684 [(set_attr "neon_type" "neon_shift_1")]
5685)
5686
5687(define_expand "vec_pack_trunc_<mode>"
5688 [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5689 (match_operand:VSHFT 1 "register_operand" "")
5690 (match_operand:VSHFT 2 "register_operand")]
b46a36c7 5691 "TARGET_NEON && !BYTES_BIG_ENDIAN"
a62cc977 5692{
5693 rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5694
5695 emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5696 emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5697 emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5698 DONE;
5699})
99c1d5bc 5700
5701(define_insn "neon_vabd<mode>_2"
5702 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5703 (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5704 (match_operand:VDQ 2 "s_register_operand" "w"))))]
5705 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5706 "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5707 [(set (attr "neon_type")
5708 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5709 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5710 (const_string "neon_fp_vadd_ddd_vabs_dd")
5711 (const_string "neon_fp_vadd_qqq_vabs_qq"))
5712 (const_string "neon_int_5")))]
5713)
5714
5715(define_insn "neon_vabd<mode>_3"
5716 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5717 (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5718 (match_operand:VDQ 2 "s_register_operand" "w")]
5719 UNSPEC_VSUB)))]
5720 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5721 "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5722 [(set (attr "neon_type")
5723 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5724 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5725 (const_string "neon_fp_vadd_ddd_vabs_dd")
5726 (const_string "neon_fp_vadd_qqq_vabs_qq"))
5727 (const_string "neon_int_5")))]
5728)
Mirror of https://gcc.gnu.org/git/gcc.git