]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/config/aarch64/aarch64-builtins.c
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix PR88046 on AArch64 and Arm bare metal targets.
[thirdparty/gcc.git] / gcc / config / aarch64 / aarch64-builtins.c
CommitLineData
43e9d192 1/* Builtins' description for AArch64 SIMD architecture.
a5544970 2 Copyright (C) 2011-2019 Free Software Foundation, Inc.
43e9d192
IB		 3 Contributed by ARM Ltd.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
8fcc61f8
RS		 21#define IN_TARGET_CODE 1
22
43e9d192
IB		 23#include "config.h"
24#include "system.h"
25#include "coretypes.h"
26#include "tm.h"
c7131fb2 27#include "function.h"
c7131fb2 28#include "basic-block.h"
e11c4407 29#include "rtl.h"
c7131fb2
AM		 30#include "tree.h"
31#include "gimple.h"
4d0cdd0c 32#include "memmodel.h"
e11c4407
AM		 33#include "tm_p.h"
34#include "expmed.h"
35#include "optabs.h"
36#include "recog.h"
37#include "diagnostic-core.h"
40e23961 38#include "fold-const.h"
d8a2d370 39#include "stor-layout.h"
36566b39 40#include "explow.h"
43e9d192 41#include "expr.h"
43e9d192 42#include "langhooks.h"
5be5c238 43#include "gimple-iterator.h"
10766209 44#include "case-cfn-macros.h"
9d63f43b 45#include "emit-rtl.h"
43e9d192 46
0d4a1197
RS		 47#define v8qi_UP E_V8QImode
48#define v4hi_UP E_V4HImode
49#define v4hf_UP E_V4HFmode
50#define v2si_UP E_V2SImode
51#define v2sf_UP E_V2SFmode
52#define v1df_UP E_V1DFmode
53#define di_UP E_DImode
54#define df_UP E_DFmode
55#define v16qi_UP E_V16QImode
56#define v8hi_UP E_V8HImode
57#define v8hf_UP E_V8HFmode
58#define v4si_UP E_V4SImode
59#define v4sf_UP E_V4SFmode
60#define v2di_UP E_V2DImode
61#define v2df_UP E_V2DFmode
62#define ti_UP E_TImode
63#define oi_UP E_OImode
64#define ci_UP E_CImode
65#define xi_UP E_XImode
66#define si_UP E_SImode
67#define sf_UP E_SFmode
68#define hi_UP E_HImode
69#define hf_UP E_HFmode
70#define qi_UP E_QImode
43e9d192
IB		 71#define UP(X) X##_UP
72
b5828b4b
JG		 73#define SIMD_MAX_BUILTIN_ARGS 5
74
75enum aarch64_type_qualifiers
43e9d192 76{
b5828b4b
JG		 77 /* T foo. */
78 qualifier_none = 0x0,
79 /* unsigned T foo. */
80 qualifier_unsigned = 0x1, /* 1 << 0 */
81 /* const T foo. */
82 qualifier_const = 0x2, /* 1 << 1 */
83 /* T *foo. */
84 qualifier_pointer = 0x4, /* 1 << 2 */
b5828b4b
JG		 85 /* Used when expanding arguments if an operand could
86 be an immediate. */
87 qualifier_immediate = 0x8, /* 1 << 3 */
88 qualifier_maybe_immediate = 0x10, /* 1 << 4 */
89 /* void foo (...). */
90 qualifier_void = 0x20, /* 1 << 5 */
91 /* Some patterns may have internal operands, this qualifier is an
92 instruction to the initialisation code to skip this operand. */
93 qualifier_internal = 0x40, /* 1 << 6 */
94 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
95 rather than using the type of the operand. */
96 qualifier_map_mode = 0x80, /* 1 << 7 */
97 /* qualifier_pointer | qualifier_map_mode */
98 qualifier_pointer_map_mode = 0x84,
e625e715 99 /* qualifier_const | qualifier_pointer | qualifier_map_mode */
6db1ec94
JG		 100 qualifier_const_pointer_map_mode = 0x86,
101 /* Polynomial types. */
2a49c16d
AL		 102 qualifier_poly = 0x100,
103 /* Lane indices - must be in range, and flipped for bigendian. */
4d0a0237
CB		 104 qualifier_lane_index = 0x200,
105 /* Lane indices for single lane structure loads and stores. */
9d63f43b
TC		 106 qualifier_struct_load_store_lane_index = 0x400,
107 /* Lane indices selected in pairs. - must be in range, and flipped for
108 bigendian. */
109 qualifier_lane_pair_index = 0x800,
b5828b4b 110};
43e9d192
IB		 111
112typedef struct
113{
114 const char *name;
ef4bddc2 115 machine_mode mode;
342be7f7
JG		 116 const enum insn_code code;
117 unsigned int fcode;
b5828b4b 118 enum aarch64_type_qualifiers *qualifiers;
43e9d192
IB		 119} aarch64_simd_builtin_datum;
120
b5828b4b
JG		 121static enum aarch64_type_qualifiers
122aarch64_types_unop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
8f905d69 123 = { qualifier_none, qualifier_none };
b5828b4b 124#define TYPES_UNOP (aarch64_types_unop_qualifiers)
5a7a4e80
TB		 125static enum aarch64_type_qualifiers
126aarch64_types_unopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
127 = { qualifier_unsigned, qualifier_unsigned };
128#define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
b5828b4b 129static enum aarch64_type_qualifiers
a579f4c7
JW		 130aarch64_types_unopus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
131 = { qualifier_unsigned, qualifier_none };
132#define TYPES_UNOPUS (aarch64_types_unopus_qualifiers)
133static enum aarch64_type_qualifiers
b5828b4b
JG		 134aarch64_types_binop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
135 = { qualifier_none, qualifier_none, qualifier_maybe_immediate };
136#define TYPES_BINOP (aarch64_types_binop_qualifiers)
137static enum aarch64_type_qualifiers
5a7a4e80
TB		 138aarch64_types_binopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
139 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned };
140#define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
7baa225d 141static enum aarch64_type_qualifiers
de10bcce
AL		 142aarch64_types_binop_uus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
143 = { qualifier_unsigned, qualifier_unsigned, qualifier_none };
144#define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
145static enum aarch64_type_qualifiers
918621d3
AL		 146aarch64_types_binop_ssu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
147 = { qualifier_none, qualifier_none, qualifier_unsigned };
148#define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
149static enum aarch64_type_qualifiers
daef0a8c
JW		 150aarch64_types_binop_uss_qualifiers[SIMD_MAX_BUILTIN_ARGS]
151 = { qualifier_unsigned, qualifier_none, qualifier_none };
152#define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers)
153static enum aarch64_type_qualifiers
7baa225d
TB		 154aarch64_types_binopp_qualifiers[SIMD_MAX_BUILTIN_ARGS]
155 = { qualifier_poly, qualifier_poly, qualifier_poly };
156#define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
157
5a7a4e80 158static enum aarch64_type_qualifiers
b5828b4b
JG		 159aarch64_types_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
160 = { qualifier_none, qualifier_none, qualifier_none, qualifier_none };
161#define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
30442682 162static enum aarch64_type_qualifiers
2a49c16d
AL		 163aarch64_types_ternop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
164 = { qualifier_none, qualifier_none, qualifier_none, qualifier_lane_index };
165#define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
166static enum aarch64_type_qualifiers
30442682
TB		 167aarch64_types_ternopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
168 = { qualifier_unsigned, qualifier_unsigned,
169 qualifier_unsigned, qualifier_unsigned };
170#define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
27086ea3
MC		 171static enum aarch64_type_qualifiers
172aarch64_types_ternopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
173 = { qualifier_unsigned, qualifier_unsigned,
174 qualifier_unsigned, qualifier_immediate };
175#define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers)
176
30442682 177
9d63f43b
TC		 178static enum aarch64_type_qualifiers
179aarch64_types_quadop_lane_pair_qualifiers[SIMD_MAX_BUILTIN_ARGS]
180 = { qualifier_none, qualifier_none, qualifier_none,
181 qualifier_none, qualifier_lane_pair_index };
182#define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers)
b5828b4b 183static enum aarch64_type_qualifiers
2a49c16d 184aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
b5828b4b 185 = { qualifier_none, qualifier_none, qualifier_none,
2a49c16d
AL		 186 qualifier_none, qualifier_lane_index };
187#define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
7a08d813
TC		 188static enum aarch64_type_qualifiers
189aarch64_types_quadopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
190 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
191 qualifier_unsigned, qualifier_lane_index };
192#define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
b5828b4b 193
27086ea3
MC		 194static enum aarch64_type_qualifiers
195aarch64_types_quadopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
196 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
197 qualifier_unsigned, qualifier_immediate };
198#define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
199
159b8724
TC		 200static enum aarch64_type_qualifiers
201aarch64_types_binop_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
202 = { qualifier_poly, qualifier_none, qualifier_immediate };
203#define TYPES_GETREGP (aarch64_types_binop_imm_p_qualifiers)
b5828b4b 204static enum aarch64_type_qualifiers
2a49c16d 205aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
b5828b4b 206 = { qualifier_none, qualifier_none, qualifier_immediate };
2a49c16d
AL		 207#define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
208#define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
b5828b4b 209static enum aarch64_type_qualifiers
de10bcce
AL		 210aarch64_types_shift_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
211 = { qualifier_unsigned, qualifier_none, qualifier_immediate };
212#define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
213static enum aarch64_type_qualifiers
1f0e9e34
JG		 214aarch64_types_fcvt_from_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
215 = { qualifier_none, qualifier_unsigned, qualifier_immediate };
216#define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers)
217static enum aarch64_type_qualifiers
252c7556
AV		 218aarch64_types_unsigned_shift_qualifiers[SIMD_MAX_BUILTIN_ARGS]
219 = { qualifier_unsigned, qualifier_unsigned, qualifier_immediate };
220#define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
de10bcce 221
252c7556 222static enum aarch64_type_qualifiers
159b8724
TC		 223aarch64_types_ternop_s_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
224 = { qualifier_none, qualifier_none, qualifier_poly, qualifier_immediate};
225#define TYPES_SETREGP (aarch64_types_ternop_s_imm_p_qualifiers)
226static enum aarch64_type_qualifiers
227aarch64_types_ternop_s_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
228 = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate};
229#define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
230#define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers)
231#define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers)
232
233static enum aarch64_type_qualifiers
234aarch64_types_ternop_p_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
235 = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_immediate};
236#define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers)
b5828b4b 237
de10bcce
AL		 238static enum aarch64_type_qualifiers
239aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS]
240 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
241 qualifier_immediate };
242#define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
243
244
b5828b4b
JG		 245static enum aarch64_type_qualifiers
246aarch64_types_combine_qualifiers[SIMD_MAX_BUILTIN_ARGS]
247 = { qualifier_none, qualifier_none, qualifier_none };
248#define TYPES_COMBINE (aarch64_types_combine_qualifiers)
249
159b8724
TC		 250static enum aarch64_type_qualifiers
251aarch64_types_combine_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
252 = { qualifier_poly, qualifier_poly, qualifier_poly };
253#define TYPES_COMBINEP (aarch64_types_combine_p_qualifiers)
254
b5828b4b
JG		 255static enum aarch64_type_qualifiers
256aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
257 = { qualifier_none, qualifier_const_pointer_map_mode };
258#define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
259#define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
3ec1be97
CB		 260static enum aarch64_type_qualifiers
261aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
262 = { qualifier_none, qualifier_const_pointer_map_mode,
4d0a0237 263 qualifier_none, qualifier_struct_load_store_lane_index };
3ec1be97 264#define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
b5828b4b 265
46e778c4
JG		 266static enum aarch64_type_qualifiers
267aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
268 = { qualifier_poly, qualifier_unsigned,
269 qualifier_poly, qualifier_poly };
270#define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
271static enum aarch64_type_qualifiers
272aarch64_types_bsl_s_qualifiers[SIMD_MAX_BUILTIN_ARGS]
273 = { qualifier_none, qualifier_unsigned,
274 qualifier_none, qualifier_none };
275#define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
276static enum aarch64_type_qualifiers
277aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
278 = { qualifier_unsigned, qualifier_unsigned,
279 qualifier_unsigned, qualifier_unsigned };
280#define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
281
b5828b4b
JG		 282/* The first argument (return type) of a store should be void type,
283 which we represent with qualifier_void. Their first operand will be
284 a DImode pointer to the location to store to, so we must use
285 qualifier_map_mode | qualifier_pointer to build a pointer to the
286 element type of the vector. */
287static enum aarch64_type_qualifiers
159b8724
TC		 288aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
289 = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
290#define TYPES_STORE1P (aarch64_types_store1_p_qualifiers)
291static enum aarch64_type_qualifiers
b5828b4b
JG		 292aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
293 = { qualifier_void, qualifier_pointer_map_mode, qualifier_none };
294#define TYPES_STORE1 (aarch64_types_store1_qualifiers)
295#define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
ba081b77
JG		 296static enum aarch64_type_qualifiers
297aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
298 = { qualifier_void, qualifier_pointer_map_mode,
4d0a0237 299 qualifier_none, qualifier_struct_load_store_lane_index };
ba081b77 300#define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
b5828b4b 301
0ddec79f
JG		 302#define CF0(N, X) CODE_FOR_aarch64_##N##X
303#define CF1(N, X) CODE_FOR_##N##X##1
304#define CF2(N, X) CODE_FOR_##N##X##2
305#define CF3(N, X) CODE_FOR_##N##X##3
306#define CF4(N, X) CODE_FOR_##N##X##4
307#define CF10(N, X) CODE_FOR_##N##X
308
309#define VAR1(T, N, MAP, A) \
bc5e395d 310 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T},
0ddec79f
JG		 311#define VAR2(T, N, MAP, A, B) \
312 VAR1 (T, N, MAP, A) \
313 VAR1 (T, N, MAP, B)
314#define VAR3(T, N, MAP, A, B, C) \
315 VAR2 (T, N, MAP, A, B) \
316 VAR1 (T, N, MAP, C)
317#define VAR4(T, N, MAP, A, B, C, D) \
318 VAR3 (T, N, MAP, A, B, C) \
319 VAR1 (T, N, MAP, D)
320#define VAR5(T, N, MAP, A, B, C, D, E) \
321 VAR4 (T, N, MAP, A, B, C, D) \
322 VAR1 (T, N, MAP, E)
323#define VAR6(T, N, MAP, A, B, C, D, E, F) \
324 VAR5 (T, N, MAP, A, B, C, D, E) \
325 VAR1 (T, N, MAP, F)
326#define VAR7(T, N, MAP, A, B, C, D, E, F, G) \
327 VAR6 (T, N, MAP, A, B, C, D, E, F) \
328 VAR1 (T, N, MAP, G)
329#define VAR8(T, N, MAP, A, B, C, D, E, F, G, H) \
330 VAR7 (T, N, MAP, A, B, C, D, E, F, G) \
331 VAR1 (T, N, MAP, H)
332#define VAR9(T, N, MAP, A, B, C, D, E, F, G, H, I) \
333 VAR8 (T, N, MAP, A, B, C, D, E, F, G, H) \
334 VAR1 (T, N, MAP, I)
335#define VAR10(T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
336 VAR9 (T, N, MAP, A, B, C, D, E, F, G, H, I) \
337 VAR1 (T, N, MAP, J)
338#define VAR11(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
339 VAR10 (T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
340 VAR1 (T, N, MAP, K)
341#define VAR12(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
342 VAR11 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
343 VAR1 (T, N, MAP, L)
7c369485
AL		 344#define VAR13(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
345 VAR12 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
346 VAR1 (T, N, MAP, M)
347#define VAR14(T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
348 VAR13 (T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
349 VAR1 (T, X, MAP, N)
342be7f7 350
f421c516 351#include "aarch64-builtin-iterators.h"
43e9d192
IB		 352
353static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
342be7f7
JG		 354#include "aarch64-simd-builtins.def"
355};
356
5d357f26
KT		 357/* There's only 8 CRC32 builtins. Probably not worth their own .def file. */
358#define AARCH64_CRC32_BUILTINS \
359 CRC32_BUILTIN (crc32b, QI) \
360 CRC32_BUILTIN (crc32h, HI) \
361 CRC32_BUILTIN (crc32w, SI) \
362 CRC32_BUILTIN (crc32x, DI) \
363 CRC32_BUILTIN (crc32cb, QI) \
364 CRC32_BUILTIN (crc32ch, HI) \
365 CRC32_BUILTIN (crc32cw, SI) \
366 CRC32_BUILTIN (crc32cx, DI)
367
9d63f43b
TC		 368/* The next 8 FCMLA instrinsics require some special handling compared the
369 normal simd intrinsics. */
370#define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \
371 FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \
372 FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \
373 FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \
374 FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \
375 FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \
376 FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \
377 FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \
378 FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \
379
5d357f26
KT		 380typedef struct
381{
382 const char *name;
ef4bddc2 383 machine_mode mode;
5d357f26
KT		 384 const enum insn_code icode;
385 unsigned int fcode;
386} aarch64_crc_builtin_datum;
387
9d63f43b
TC		 388/* Hold information about how to expand the FCMLA_LANEQ builtins. */
389typedef struct
390{
391 const char *name;
392 machine_mode mode;
393 const enum insn_code icode;
394 unsigned int fcode;
395 bool lane;
396} aarch64_fcmla_laneq_builtin_datum;
397
5d357f26
KT		 398#define CRC32_BUILTIN(N, M) \
399 AARCH64_BUILTIN_##N,
400
9d63f43b
TC		 401#define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
402 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M,
403
342be7f7 404#undef VAR1
0ddec79f 405#define VAR1(T, N, MAP, A) \
e993fea1 406 AARCH64_SIMD_BUILTIN_##T##_##N##A,
342be7f7
JG		 407
408enum aarch64_builtins
409{
410 AARCH64_BUILTIN_MIN,
aa87aced
KV		 411
412 AARCH64_BUILTIN_GET_FPCR,
413 AARCH64_BUILTIN_SET_FPCR,
414 AARCH64_BUILTIN_GET_FPSR,
415 AARCH64_BUILTIN_SET_FPSR,
416
a6fc00da
BH		 417 AARCH64_BUILTIN_RSQRT_DF,
418 AARCH64_BUILTIN_RSQRT_SF,
419 AARCH64_BUILTIN_RSQRT_V2DF,
420 AARCH64_BUILTIN_RSQRT_V2SF,
421 AARCH64_BUILTIN_RSQRT_V4SF,
342be7f7 422 AARCH64_SIMD_BUILTIN_BASE,
661fce82 423 AARCH64_SIMD_BUILTIN_LANE_CHECK,
342be7f7 424#include "aarch64-simd-builtins.def"
661fce82
AL		 425 /* The first enum element which is based on an insn_data pattern. */
426 AARCH64_SIMD_PATTERN_START = AARCH64_SIMD_BUILTIN_LANE_CHECK + 1,
427 AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_PATTERN_START
428 + ARRAY_SIZE (aarch64_simd_builtin_data) - 1,
5d357f26
KT		 429 AARCH64_CRC32_BUILTIN_BASE,
430 AARCH64_CRC32_BUILTINS
431 AARCH64_CRC32_BUILTIN_MAX,
312492bd
JW		 432 /* ARMv8.3-A Pointer Authentication Builtins. */
433 AARCH64_PAUTH_BUILTIN_AUTIA1716,
434 AARCH64_PAUTH_BUILTIN_PACIA1716,
435 AARCH64_PAUTH_BUILTIN_XPACLRI,
9d63f43b
TC		 436 /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins. */
437 AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE,
438 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
342be7f7 439 AARCH64_BUILTIN_MAX
43e9d192
IB		 440};
441
5d357f26
KT		 442#undef CRC32_BUILTIN
443#define CRC32_BUILTIN(N, M) \
0d4a1197 444 {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
5d357f26
KT		 445
446static aarch64_crc_builtin_datum aarch64_crc_builtin_data[] = {
447 AARCH64_CRC32_BUILTINS
448};
449
9d63f43b
TC		 450
451#undef FCMLA_LANEQ_BUILTIN
452#define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
453 {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \
454 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T},
455
456/* This structure contains how to manage the mapping form the builtin to the
457 instruction to generate in the backend and how to invoke the instruction. */
458static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data[] {
459 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
460};
461
5d357f26
KT		 462#undef CRC32_BUILTIN
463
119103ca
JG		 464static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX];
465
43e9d192
IB		 466#define NUM_DREG_TYPES 6
467#define NUM_QREG_TYPES 6
468
f9d53c27
TB		 469/* Internal scalar builtin types. These types are used to support
470 neon intrinsic builtins. They are _not_ user-visible types. Therefore
471 the mangling for these types are implementation defined. */
472const char *aarch64_scalar_builtin_types[] = {
473 "__builtin_aarch64_simd_qi",
474 "__builtin_aarch64_simd_hi",
475 "__builtin_aarch64_simd_si",
7c369485 476 "__builtin_aarch64_simd_hf",
f9d53c27
TB		 477 "__builtin_aarch64_simd_sf",
478 "__builtin_aarch64_simd_di",
479 "__builtin_aarch64_simd_df",
480 "__builtin_aarch64_simd_poly8",
481 "__builtin_aarch64_simd_poly16",
482 "__builtin_aarch64_simd_poly64",
483 "__builtin_aarch64_simd_poly128",
484 "__builtin_aarch64_simd_ti",
485 "__builtin_aarch64_simd_uqi",
486 "__builtin_aarch64_simd_uhi",
487 "__builtin_aarch64_simd_usi",
488 "__builtin_aarch64_simd_udi",
489 "__builtin_aarch64_simd_ei",
490 "__builtin_aarch64_simd_oi",
491 "__builtin_aarch64_simd_ci",
492 "__builtin_aarch64_simd_xi",
493 NULL
494};
b5828b4b 495
f9d53c27
TB		 496#define ENTRY(E, M, Q, G) E,
497enum aarch64_simd_type
498{
499#include "aarch64-simd-builtin-types.def"
500 ARM_NEON_H_TYPES_LAST
501};
502#undef ENTRY
b5828b4b 503
f9d53c27 504struct aarch64_simd_type_info
b5828b4b 505{
f9d53c27
TB		 506 enum aarch64_simd_type type;
507
508 /* Internal type name. */
509 const char *name;
510
511 /* Internal type name(mangled). The mangled names conform to the
512 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
513 Appendix A). To qualify for emission with the mangled names defined in
514 that document, a vector type must not only be of the correct mode but also
515 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
516 types are registered by aarch64_init_simd_builtin_types (). In other
517 words, vector types defined in other ways e.g. via vector_size attribute
518 will get default mangled names. */
519 const char *mangle;
520
521 /* Internal type. */
522 tree itype;
523
524 /* Element type. */
b5828b4b
JG		 525 tree eltype;
526
f9d53c27
TB		 527 /* Machine mode the internal type maps to. */
528 enum machine_mode mode;
b5828b4b 529
f9d53c27
TB		 530 /* Qualifiers. */
531 enum aarch64_type_qualifiers q;
532};
533
534#define ENTRY(E, M, Q, G) \
0d4a1197 535 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
f9d53c27
TB		 536static struct aarch64_simd_type_info aarch64_simd_types [] = {
537#include "aarch64-simd-builtin-types.def"
538};
539#undef ENTRY
540
541static tree aarch64_simd_intOI_type_node = NULL_TREE;
f9d53c27
TB		 542static tree aarch64_simd_intCI_type_node = NULL_TREE;
543static tree aarch64_simd_intXI_type_node = NULL_TREE;
544
1b62ed4f
JG		 545/* The user-visible __fp16 type, and a pointer to that type. Used
546 across the back-end. */
547tree aarch64_fp16_type_node = NULL_TREE;
548tree aarch64_fp16_ptr_type_node = NULL_TREE;
549
f9d53c27
TB		 550static const char *
551aarch64_mangle_builtin_scalar_type (const_tree type)
552{
553 int i = 0;
554
555 while (aarch64_scalar_builtin_types[i] != NULL)
b5828b4b 556 {
f9d53c27
TB		 557 const char *name = aarch64_scalar_builtin_types[i];
558
559 if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
560 && DECL_NAME (TYPE_NAME (type))
561 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), name))
562 return aarch64_scalar_builtin_types[i];
563 i++;
564 }
565 return NULL;
b5828b4b
JG		 566}
567
f9d53c27
TB		 568static const char *
569aarch64_mangle_builtin_vector_type (const_tree type)
b5828b4b 570{
f9d53c27
TB		 571 int i;
572 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
573
574 for (i = 0; i < nelts; i++)
575 if (aarch64_simd_types[i].mode == TYPE_MODE (type)
576 && TYPE_NAME (type)
577 && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
578 && DECL_NAME (TYPE_NAME (type))
579 && !strcmp
580 (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))),
581 aarch64_simd_types[i].name))
582 return aarch64_simd_types[i].mangle;
583
584 return NULL;
6db1ec94
JG		 585}
586
f9d53c27
TB		 587const char *
588aarch64_mangle_builtin_type (const_tree type)
6db1ec94 589{
f9d53c27
TB		 590 const char *mangle;
591 /* Walk through all the AArch64 builtins types tables to filter out the
592 incoming type. */
593 if ((mangle = aarch64_mangle_builtin_vector_type (type))
594 || (mangle = aarch64_mangle_builtin_scalar_type (type)))
595 return mangle;
596
597 return NULL;
6db1ec94
JG		 598}
599
f9d53c27 600static tree
b8506a8a 601aarch64_simd_builtin_std_type (machine_mode mode,
f9d53c27 602 enum aarch64_type_qualifiers q)
6db1ec94 603{
f9d53c27
TB		 604#define QUAL_TYPE(M) \
605 ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node);
606 switch (mode)
607 {
4e10a5a7 608 case E_QImode:
f9d53c27 609 return QUAL_TYPE (QI);
4e10a5a7 610 case E_HImode:
f9d53c27 611 return QUAL_TYPE (HI);
4e10a5a7 612 case E_SImode:
f9d53c27 613 return QUAL_TYPE (SI);
4e10a5a7 614 case E_DImode:
f9d53c27 615 return QUAL_TYPE (DI);
4e10a5a7 616 case E_TImode:
f9d53c27 617 return QUAL_TYPE (TI);
4e10a5a7 618 case E_OImode:
f9d53c27 619 return aarch64_simd_intOI_type_node;
4e10a5a7 620 case E_CImode:
f9d53c27 621 return aarch64_simd_intCI_type_node;
4e10a5a7 622 case E_XImode:
f9d53c27 623 return aarch64_simd_intXI_type_node;
4e10a5a7 624 case E_HFmode:
71a11456 625 return aarch64_fp16_type_node;
4e10a5a7 626 case E_SFmode:
f9d53c27 627 return float_type_node;
4e10a5a7 628 case E_DFmode:
f9d53c27
TB		 629 return double_type_node;
630 default:
631 gcc_unreachable ();
632 }
633#undef QUAL_TYPE
6db1ec94
JG		 634}
635
f9d53c27 636static tree
b8506a8a 637aarch64_lookup_simd_builtin_type (machine_mode mode,
f9d53c27 638 enum aarch64_type_qualifiers q)
6db1ec94 639{
f9d53c27
TB		 640 int i;
641 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
642
643 /* Non-poly scalar modes map to standard types not in the table. */
644 if (q != qualifier_poly && !VECTOR_MODE_P (mode))
645 return aarch64_simd_builtin_std_type (mode, q);
646
647 for (i = 0; i < nelts; i++)
648 if (aarch64_simd_types[i].mode == mode
649 && aarch64_simd_types[i].q == q)
650 return aarch64_simd_types[i].itype;
651
652 return NULL_TREE;
b5828b4b
JG		 653}
654
f9d53c27 655static tree
b8506a8a 656aarch64_simd_builtin_type (machine_mode mode,
f9d53c27
TB		 657 bool unsigned_p, bool poly_p)
658{
659 if (poly_p)
660 return aarch64_lookup_simd_builtin_type (mode, qualifier_poly);
661 else if (unsigned_p)
662 return aarch64_lookup_simd_builtin_type (mode, qualifier_unsigned);
663 else
664 return aarch64_lookup_simd_builtin_type (mode, qualifier_none);
665}
666
af55e82d 667static void
f9d53c27 668aarch64_init_simd_builtin_types (void)
43e9d192 669{
f9d53c27
TB		 670 int i;
671 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
672 tree tdecl;
673
674 /* Init all the element types built by the front-end. */
675 aarch64_simd_types[Int8x8_t].eltype = intQI_type_node;
676 aarch64_simd_types[Int8x16_t].eltype = intQI_type_node;
677 aarch64_simd_types[Int16x4_t].eltype = intHI_type_node;
678 aarch64_simd_types[Int16x8_t].eltype = intHI_type_node;
679 aarch64_simd_types[Int32x2_t].eltype = intSI_type_node;
680 aarch64_simd_types[Int32x4_t].eltype = intSI_type_node;
681 aarch64_simd_types[Int64x1_t].eltype = intDI_type_node;
682 aarch64_simd_types[Int64x2_t].eltype = intDI_type_node;
683 aarch64_simd_types[Uint8x8_t].eltype = unsigned_intQI_type_node;
684 aarch64_simd_types[Uint8x16_t].eltype = unsigned_intQI_type_node;
685 aarch64_simd_types[Uint16x4_t].eltype = unsigned_intHI_type_node;
686 aarch64_simd_types[Uint16x8_t].eltype = unsigned_intHI_type_node;
687 aarch64_simd_types[Uint32x2_t].eltype = unsigned_intSI_type_node;
688 aarch64_simd_types[Uint32x4_t].eltype = unsigned_intSI_type_node;
689 aarch64_simd_types[Uint64x1_t].eltype = unsigned_intDI_type_node;
690 aarch64_simd_types[Uint64x2_t].eltype = unsigned_intDI_type_node;
691
692 /* Poly types are a world of their own. */
693 aarch64_simd_types[Poly8_t].eltype = aarch64_simd_types[Poly8_t].itype =
694 build_distinct_type_copy (unsigned_intQI_type_node);
bcee52c4
MS		 695 /* Prevent front-ends from transforming Poly8_t arrays into string
696 literals. */
697 TYPE_STRING_FLAG (aarch64_simd_types[Poly8_t].eltype) = false;
698
f9d53c27
TB		 699 aarch64_simd_types[Poly16_t].eltype = aarch64_simd_types[Poly16_t].itype =
700 build_distinct_type_copy (unsigned_intHI_type_node);
701 aarch64_simd_types[Poly64_t].eltype = aarch64_simd_types[Poly64_t].itype =
702 build_distinct_type_copy (unsigned_intDI_type_node);
703 aarch64_simd_types[Poly128_t].eltype = aarch64_simd_types[Poly128_t].itype =
704 build_distinct_type_copy (unsigned_intTI_type_node);
705 /* Init poly vector element types with scalar poly types. */
706 aarch64_simd_types[Poly8x8_t].eltype = aarch64_simd_types[Poly8_t].itype;
707 aarch64_simd_types[Poly8x16_t].eltype = aarch64_simd_types[Poly8_t].itype;
708 aarch64_simd_types[Poly16x4_t].eltype = aarch64_simd_types[Poly16_t].itype;
709 aarch64_simd_types[Poly16x8_t].eltype = aarch64_simd_types[Poly16_t].itype;
710 aarch64_simd_types[Poly64x1_t].eltype = aarch64_simd_types[Poly64_t].itype;
711 aarch64_simd_types[Poly64x2_t].eltype = aarch64_simd_types[Poly64_t].itype;
712
713 /* Continue with standard types. */
71a11456
AL		 714 aarch64_simd_types[Float16x4_t].eltype = aarch64_fp16_type_node;
715 aarch64_simd_types[Float16x8_t].eltype = aarch64_fp16_type_node;
f9d53c27
TB		 716 aarch64_simd_types[Float32x2_t].eltype = float_type_node;
717 aarch64_simd_types[Float32x4_t].eltype = float_type_node;
718 aarch64_simd_types[Float64x1_t].eltype = double_type_node;
719 aarch64_simd_types[Float64x2_t].eltype = double_type_node;
720
721 for (i = 0; i < nelts; i++)
722 {
723 tree eltype = aarch64_simd_types[i].eltype;
b8506a8a 724 machine_mode mode = aarch64_simd_types[i].mode;
f9d53c27
TB		 725
726 if (aarch64_simd_types[i].itype == NULL)
b96824c4
RFI		 727 {
728 aarch64_simd_types[i].itype
729 = build_distinct_type_copy
730 (build_vector_type (eltype, GET_MODE_NUNITS (mode)));
731 SET_TYPE_STRUCTURAL_EQUALITY (aarch64_simd_types[i].itype);
732 }
f9d53c27
TB		 733
734 tdecl = add_builtin_type (aarch64_simd_types[i].name,
735 aarch64_simd_types[i].itype);
736 TYPE_NAME (aarch64_simd_types[i].itype) = tdecl;
f9d53c27 737 }
43e9d192 738
f9d53c27
TB		 739#define AARCH64_BUILD_SIGNED_TYPE(mode) \
740 make_signed_type (GET_MODE_PRECISION (mode));
741 aarch64_simd_intOI_type_node = AARCH64_BUILD_SIGNED_TYPE (OImode);
f9d53c27
TB		 742 aarch64_simd_intCI_type_node = AARCH64_BUILD_SIGNED_TYPE (CImode);
743 aarch64_simd_intXI_type_node = AARCH64_BUILD_SIGNED_TYPE (XImode);
744#undef AARCH64_BUILD_SIGNED_TYPE
745
f9d53c27
TB		 746 tdecl = add_builtin_type
747 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node);
748 TYPE_NAME (aarch64_simd_intOI_type_node) = tdecl;
749 tdecl = add_builtin_type
750 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node);
751 TYPE_NAME (aarch64_simd_intCI_type_node) = tdecl;
752 tdecl = add_builtin_type
753 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node);
754 TYPE_NAME (aarch64_simd_intXI_type_node) = tdecl;
755}
756
757static void
758aarch64_init_simd_builtin_scalar_types (void)
759{
760 /* Define typedefs for all the standard scalar types. */
761 (*lang_hooks.types.register_builtin_type) (intQI_type_node,
43e9d192 762 "__builtin_aarch64_simd_qi");
f9d53c27 763 (*lang_hooks.types.register_builtin_type) (intHI_type_node,
43e9d192 764 "__builtin_aarch64_simd_hi");
7c369485
AL		 765 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node,
766 "__builtin_aarch64_simd_hf");
f9d53c27 767 (*lang_hooks.types.register_builtin_type) (intSI_type_node,
43e9d192 768 "__builtin_aarch64_simd_si");
f9d53c27 769 (*lang_hooks.types.register_builtin_type) (float_type_node,
43e9d192 770 "__builtin_aarch64_simd_sf");
f9d53c27 771 (*lang_hooks.types.register_builtin_type) (intDI_type_node,
43e9d192 772 "__builtin_aarch64_simd_di");
f9d53c27 773 (*lang_hooks.types.register_builtin_type) (double_type_node,
43e9d192 774 "__builtin_aarch64_simd_df");
f9d53c27 775 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
43e9d192 776 "__builtin_aarch64_simd_poly8");
f9d53c27 777 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
43e9d192 778 "__builtin_aarch64_simd_poly16");
f9d53c27 779 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
7baa225d 780 "__builtin_aarch64_simd_poly64");
f9d53c27 781 (*lang_hooks.types.register_builtin_type) (unsigned_intTI_type_node,
7baa225d 782 "__builtin_aarch64_simd_poly128");
f9d53c27 783 (*lang_hooks.types.register_builtin_type) (intTI_type_node,
43e9d192 784 "__builtin_aarch64_simd_ti");
b5828b4b 785 /* Unsigned integer types for various mode sizes. */
f9d53c27 786 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
b5828b4b 787 "__builtin_aarch64_simd_uqi");
f9d53c27 788 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
b5828b4b 789 "__builtin_aarch64_simd_uhi");
f9d53c27 790 (*lang_hooks.types.register_builtin_type) (unsigned_intSI_type_node,
b5828b4b 791 "__builtin_aarch64_simd_usi");
f9d53c27 792 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
b5828b4b 793 "__builtin_aarch64_simd_udi");
f9d53c27
TB		 794}
795
e95a988a
KT		 796static bool aarch64_simd_builtins_initialized_p = false;
797
9d63f43b
TC		 798/* Due to the architecture not providing lane variant of the lane instructions
799 for fcmla we can't use the standard simd builtin expansion code, but we
800 still want the majority of the validation that would normally be done. */
801
802void
803aarch64_init_fcmla_laneq_builtins (void)
804{
805 unsigned int i = 0;
806
807 for (i = 0; i < ARRAY_SIZE (aarch64_fcmla_lane_builtin_data); ++i)
808 {
809 aarch64_fcmla_laneq_builtin_datum* d
810 = &aarch64_fcmla_lane_builtin_data[i];
811 tree argtype = aarch64_lookup_simd_builtin_type (d->mode, qualifier_none);
812 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
813 tree quadtype
814 = aarch64_lookup_simd_builtin_type (quadmode, qualifier_none);
815 tree lanetype
816 = aarch64_simd_builtin_std_type (SImode, qualifier_lane_pair_index);
817 tree ftype = build_function_type_list (argtype, argtype, argtype,
818 quadtype, lanetype, NULL_TREE);
819 tree fndecl = add_builtin_function (d->name, ftype, d->fcode,
820 BUILT_IN_MD, NULL, NULL_TREE);
821
822 aarch64_builtin_decls[d->fcode] = fndecl;
823 }
824}
825
e95a988a 826void
f9d53c27
TB		 827aarch64_init_simd_builtins (void)
828{
661fce82 829 unsigned int i, fcode = AARCH64_SIMD_PATTERN_START;
f9d53c27 830
e95a988a
KT		 831 if (aarch64_simd_builtins_initialized_p)
832 return;
833
834 aarch64_simd_builtins_initialized_p = true;
835
f9d53c27 836 aarch64_init_simd_builtin_types ();
43e9d192 837
f9d53c27
TB		 838 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
839 Therefore we need to preserve the old __builtin scalar types. It can be
840 removed once all the intrinsics become strongly typed using the qualifier
841 system. */
842 aarch64_init_simd_builtin_scalar_types ();
843
661fce82 844 tree lane_check_fpr = build_function_type_list (void_type_node,
9c4f25cc
AP		 845 size_type_node,
846 size_type_node,
661fce82
AL		 847 intSI_type_node,
848 NULL);
849 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK] =
850 add_builtin_function ("__builtin_aarch64_im_lane_boundsi", lane_check_fpr,
851 AARCH64_SIMD_BUILTIN_LANE_CHECK, BUILT_IN_MD,
852 NULL, NULL_TREE);
853
342be7f7 854 for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
43e9d192 855 {
b5828b4b 856 bool print_type_signature_p = false;
cae83731 857 char type_signature[SIMD_MAX_BUILTIN_ARGS + 1] = { 0 };
43e9d192 858 aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i];
342be7f7
JG		 859 char namebuf[60];
860 tree ftype = NULL;
119103ca 861 tree fndecl = NULL;
342be7f7 862
342be7f7 863 d->fcode = fcode;
43e9d192 864
b5828b4b
JG		 865 /* We must track two variables here. op_num is
866 the operand number as in the RTL pattern. This is
867 required to access the mode (e.g. V4SF mode) of the
868 argument, from which the base type can be derived.
869 arg_num is an index in to the qualifiers data, which
870 gives qualifiers to the type (e.g. const unsigned).
871 The reason these two variables may differ by one is the
872 void return type. While all return types take the 0th entry
873 in the qualifiers array, there is no operand for them in the
874 RTL pattern. */
875 int op_num = insn_data[d->code].n_operands - 1;
876 int arg_num = d->qualifiers[0] & qualifier_void
877 ? op_num + 1
878 : op_num;
879 tree return_type = void_type_node, args = void_list_node;
880 tree eltype;
881
882 /* Build a function type directly from the insn_data for this
883 builtin. The build_function_type () function takes care of
884 removing duplicates for us. */
885 for (; op_num >= 0; arg_num--, op_num--)
43e9d192 886 {
ef4bddc2 887 machine_mode op_mode = insn_data[d->code].operand[op_num].mode;
b5828b4b 888 enum aarch64_type_qualifiers qualifiers = d->qualifiers[arg_num];
43e9d192 889
b5828b4b
JG		 890 if (qualifiers & qualifier_unsigned)
891 {
9fd2074d 892 type_signature[op_num] = 'u';
b5828b4b
JG		 893 print_type_signature_p = true;
894 }
6db1ec94
JG		 895 else if (qualifiers & qualifier_poly)
896 {
9fd2074d 897 type_signature[op_num] = 'p';
6db1ec94
JG		 898 print_type_signature_p = true;
899 }
b5828b4b 900 else
9fd2074d 901 type_signature[op_num] = 's';
b5828b4b
JG		 902
903 /* Skip an internal operand for vget_{low, high}. */
904 if (qualifiers & qualifier_internal)
905 continue;
906
907 /* Some builtins have different user-facing types
908 for certain arguments, encoded in d->mode. */
909 if (qualifiers & qualifier_map_mode)
bc5e395d 910 op_mode = d->mode;
b5828b4b
JG		 911
912 /* For pointers, we want a pointer to the basic type
913 of the vector. */
914 if (qualifiers & qualifier_pointer && VECTOR_MODE_P (op_mode))
915 op_mode = GET_MODE_INNER (op_mode);
916
f9d53c27
TB		 917 eltype = aarch64_simd_builtin_type
918 (op_mode,
919 (qualifiers & qualifier_unsigned) != 0,
920 (qualifiers & qualifier_poly) != 0);
921 gcc_assert (eltype != NULL);
b5828b4b
JG		 922
923 /* Add qualifiers. */
924 if (qualifiers & qualifier_const)
925 eltype = build_qualified_type (eltype, TYPE_QUAL_CONST);
926
927 if (qualifiers & qualifier_pointer)
928 eltype = build_pointer_type (eltype);
929
930 /* If we have reached arg_num == 0, we are at a non-void
931 return type. Otherwise, we are still processing
932 arguments. */
933 if (arg_num == 0)
934 return_type = eltype;
935 else
936 args = tree_cons (NULL_TREE, eltype, args);
937 }
342be7f7 938
b5828b4b 939 ftype = build_function_type (return_type, args);
43e9d192 940
342be7f7 941 gcc_assert (ftype != NULL);
43e9d192 942
b5828b4b 943 if (print_type_signature_p)
bc5e395d
JG		 944 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s_%s",
945 d->name, type_signature);
b5828b4b 946 else
bc5e395d
JG		 947 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s",
948 d->name);
43e9d192 949
119103ca
JG		 950 fndecl = add_builtin_function (namebuf, ftype, fcode, BUILT_IN_MD,
951 NULL, NULL_TREE);
952 aarch64_builtin_decls[fcode] = fndecl;
43e9d192 953 }
280d970b
TC		 954
955 /* Initialize the remaining fcmla_laneq intrinsics. */
956 aarch64_init_fcmla_laneq_builtins ();
43e9d192
IB		 957}
958
5d357f26
KT		 959static void
960aarch64_init_crc32_builtins ()
961{
f9d53c27 962 tree usi_type = aarch64_simd_builtin_std_type (SImode, qualifier_unsigned);
5d357f26
KT		 963 unsigned int i = 0;
964
965 for (i = 0; i < ARRAY_SIZE (aarch64_crc_builtin_data); ++i)
966 {
967 aarch64_crc_builtin_datum* d = &aarch64_crc_builtin_data[i];
f9d53c27
TB		 968 tree argtype = aarch64_simd_builtin_std_type (d->mode,
969 qualifier_unsigned);
5d357f26
KT		 970 tree ftype = build_function_type_list (usi_type, usi_type, argtype, NULL_TREE);
971 tree fndecl = add_builtin_function (d->name, ftype, d->fcode,
972 BUILT_IN_MD, NULL, NULL_TREE);
973
974 aarch64_builtin_decls[d->fcode] = fndecl;
975 }
976}
977
a6fc00da
BH		 978/* Add builtins for reciprocal square root. */
979
980void
981aarch64_init_builtin_rsqrt (void)
982{
983 tree fndecl = NULL;
984 tree ftype = NULL;
985
986 tree V2SF_type_node = build_vector_type (float_type_node, 2);
987 tree V2DF_type_node = build_vector_type (double_type_node, 2);
988 tree V4SF_type_node = build_vector_type (float_type_node, 4);
989
990 struct builtin_decls_data
991 {
992 tree type_node;
993 const char *builtin_name;
994 int function_code;
995 };
996
997 builtin_decls_data bdda[] =
998 {
999 { double_type_node, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF },
1000 { float_type_node, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF },
1001 { V2DF_type_node, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF },
1002 { V2SF_type_node, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF },
1003 { V4SF_type_node, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF }
1004 };
1005
1006 builtin_decls_data *bdd = bdda;
1007 builtin_decls_data *bdd_end = bdd + (sizeof (bdda) / sizeof (builtin_decls_data));
1008
1009 for (; bdd < bdd_end; bdd++)
1010 {
1011 ftype = build_function_type_list (bdd->type_node, bdd->type_node, NULL_TREE);
1012 fndecl = add_builtin_function (bdd->builtin_name,
1013 ftype, bdd->function_code, BUILT_IN_MD, NULL, NULL_TREE);
1014 aarch64_builtin_decls[bdd->function_code] = fndecl;
1015 }
1016}
1017
1b62ed4f
JG		 1018/* Initialize the backend types that support the user-visible __fp16
1019 type, also initialize a pointer to that type, to be used when
1020 forming HFAs. */
1021
1022static void
1023aarch64_init_fp16_types (void)
1024{
1025 aarch64_fp16_type_node = make_node (REAL_TYPE);
1026 TYPE_PRECISION (aarch64_fp16_type_node) = 16;
1027 layout_type (aarch64_fp16_type_node);
1028
1029 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node, "__fp16");
1030 aarch64_fp16_ptr_type_node = build_pointer_type (aarch64_fp16_type_node);
1031}
1032
312492bd
JW		 1033/* Pointer authentication builtins that will become NOP on legacy platform.
1034 Currently, these builtins are for internal use only (libgcc EH unwinder). */
1035
1036void
1037aarch64_init_pauth_hint_builtins (void)
1038{
1039 /* Pointer Authentication builtins. */
1040 tree ftype_pointer_auth
1041 = build_function_type_list (ptr_type_node, ptr_type_node,
1042 unsigned_intDI_type_node, NULL_TREE);
1043 tree ftype_pointer_strip
1044 = build_function_type_list (ptr_type_node, ptr_type_node, NULL_TREE);
1045
1046 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIA1716]
1047 = add_builtin_function ("__builtin_aarch64_autia1716", ftype_pointer_auth,
1048 AARCH64_PAUTH_BUILTIN_AUTIA1716, BUILT_IN_MD, NULL,
1049 NULL_TREE);
1050 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIA1716]
1051 = add_builtin_function ("__builtin_aarch64_pacia1716", ftype_pointer_auth,
1052 AARCH64_PAUTH_BUILTIN_PACIA1716, BUILT_IN_MD, NULL,
1053 NULL_TREE);
1054 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_XPACLRI]
1055 = add_builtin_function ("__builtin_aarch64_xpaclri", ftype_pointer_strip,
1056 AARCH64_PAUTH_BUILTIN_XPACLRI, BUILT_IN_MD, NULL,
1057 NULL_TREE);
1058}
1059
342be7f7
JG		 1060void
1061aarch64_init_builtins (void)
43e9d192 1062{
aa87aced
KV		 1063 tree ftype_set_fpr
1064 = build_function_type_list (void_type_node, unsigned_type_node, NULL);
1065 tree ftype_get_fpr
1066 = build_function_type_list (unsigned_type_node, NULL);
1067
1068 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR]
1069 = add_builtin_function ("__builtin_aarch64_get_fpcr", ftype_get_fpr,
1070 AARCH64_BUILTIN_GET_FPCR, BUILT_IN_MD, NULL, NULL_TREE);
1071 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR]
1072 = add_builtin_function ("__builtin_aarch64_set_fpcr", ftype_set_fpr,
1073 AARCH64_BUILTIN_SET_FPCR, BUILT_IN_MD, NULL, NULL_TREE);
1074 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR]
1075 = add_builtin_function ("__builtin_aarch64_get_fpsr", ftype_get_fpr,
1076 AARCH64_BUILTIN_GET_FPSR, BUILT_IN_MD, NULL, NULL_TREE);
1077 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR]
1078 = add_builtin_function ("__builtin_aarch64_set_fpsr", ftype_set_fpr,
1079 AARCH64_BUILTIN_SET_FPSR, BUILT_IN_MD, NULL, NULL_TREE);
1080
1b62ed4f 1081 aarch64_init_fp16_types ();
c2ec330c 1082
342be7f7 1083 if (TARGET_SIMD)
280d970b 1084 aarch64_init_simd_builtins ();
e95a988a
KT		 1085
1086 aarch64_init_crc32_builtins ();
a6fc00da 1087 aarch64_init_builtin_rsqrt ();
312492bd 1088
a876231c
JW		 1089 /* Initialize pointer authentication builtins which are backed by instructions
1090 in NOP encoding space.
1091
1092 NOTE: these builtins are supposed to be used by libgcc unwinder only, as
1093 there is no support on return address signing under ILP32, we don't
1094 register them. */
1095 if (!TARGET_ILP32)
1096 aarch64_init_pauth_hint_builtins ();
43e9d192
IB		 1097}
1098
119103ca
JG		 1099tree
1100aarch64_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED)
1101{
1102 if (code >= AARCH64_BUILTIN_MAX)
1103 return error_mark_node;
1104
1105 return aarch64_builtin_decls[code];
1106}
1107
43e9d192
IB		 1108typedef enum
1109{
1110 SIMD_ARG_COPY_TO_REG,
1111 SIMD_ARG_CONSTANT,
2a49c16d 1112 SIMD_ARG_LANE_INDEX,
4d0a0237 1113 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX,
9d63f43b 1114 SIMD_ARG_LANE_PAIR_INDEX,
43e9d192
IB		 1115 SIMD_ARG_STOP
1116} builtin_simd_arg;
1117
e95a988a 1118
43e9d192
IB		 1119static rtx
1120aarch64_simd_expand_args (rtx target, int icode, int have_retval,
4d0a0237 1121 tree exp, builtin_simd_arg *args,
b8506a8a 1122 machine_mode builtin_mode)
43e9d192 1123{
43e9d192 1124 rtx pat;
d9e80f49
AL		 1125 rtx op[SIMD_MAX_BUILTIN_ARGS + 1]; /* First element for result operand. */
1126 int opc = 0;
1127
1128 if (have_retval)
1129 {
1130 machine_mode tmode = insn_data[icode].operand[0].mode;
1131 if (!target
43e9d192 1132 || GET_MODE (target) != tmode
d9e80f49
AL		 1133 || !(*insn_data[icode].operand[0].predicate) (target, tmode))
1134 target = gen_reg_rtx (tmode);
1135 op[opc++] = target;
1136 }
43e9d192 1137
43e9d192
IB		 1138 for (;;)
1139 {
d9e80f49 1140 builtin_simd_arg thisarg = args[opc - have_retval];
43e9d192
IB		 1141
1142 if (thisarg == SIMD_ARG_STOP)
1143 break;
1144 else
1145 {
d9e80f49 1146 tree arg = CALL_EXPR_ARG (exp, opc - have_retval);
b8506a8a 1147 machine_mode mode = insn_data[icode].operand[opc].mode;
d9e80f49 1148 op[opc] = expand_normal (arg);
43e9d192
IB		 1149
1150 switch (thisarg)
1151 {
1152 case SIMD_ARG_COPY_TO_REG:
d9e80f49
AL		 1153 if (POINTER_TYPE_P (TREE_TYPE (arg)))
1154 op[opc] = convert_memory_address (Pmode, op[opc]);
1155 /*gcc_assert (GET_MODE (op[opc]) == mode); */
1156 if (!(*insn_data[icode].operand[opc].predicate)
1157 (op[opc], mode))
1158 op[opc] = copy_to_mode_reg (mode, op[opc]);
43e9d192
IB		 1159 break;
1160
4d0a0237
CB		 1161 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX:
1162 gcc_assert (opc > 1);
1163 if (CONST_INT_P (op[opc]))
1164 {
6a70badb
RS		 1165 unsigned int nunits
1166 = GET_MODE_NUNITS (builtin_mode).to_constant ();
1167 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
4d0a0237 1168 /* Keep to GCC-vector-extension lane indices in the RTL. */
7ac29c0f
RS		 1169 op[opc] = aarch64_endian_lane_rtx (builtin_mode,
1170 INTVAL (op[opc]));
4d0a0237
CB		 1171 }
1172 goto constant_arg;
1173
2a49c16d
AL		 1174 case SIMD_ARG_LANE_INDEX:
1175 /* Must be a previous operand into which this is an index. */
d9e80f49
AL		 1176 gcc_assert (opc > 0);
1177 if (CONST_INT_P (op[opc]))
2a49c16d 1178 {
d9e80f49 1179 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
6a70badb
RS		 1180 unsigned int nunits
1181 = GET_MODE_NUNITS (vmode).to_constant ();
1182 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
2a49c16d 1183 /* Keep to GCC-vector-extension lane indices in the RTL. */
7ac29c0f 1184 op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc]));
2a49c16d 1185 }
9d63f43b
TC		 1186 /* If the lane index isn't a constant then error out. */
1187 goto constant_arg;
1188
1189 case SIMD_ARG_LANE_PAIR_INDEX:
1190 /* Must be a previous operand into which this is an index and
1191 index is restricted to nunits / 2. */
1192 gcc_assert (opc > 0);
1193 if (CONST_INT_P (op[opc]))
1194 {
1195 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1196 unsigned int nunits
1197 = GET_MODE_NUNITS (vmode).to_constant ();
1198 aarch64_simd_lane_bounds (op[opc], 0, nunits / 2, exp);
1199 /* Keep to GCC-vector-extension lane indices in the RTL. */
1200 op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc]));
1201 }
2a49c16d
AL		 1202 /* Fall through - if the lane index isn't a constant then
1203 the next case will error. */
191816a3 1204 /* FALLTHRU */
43e9d192 1205 case SIMD_ARG_CONSTANT:
4d0a0237 1206constant_arg:
d9e80f49
AL		 1207 if (!(*insn_data[icode].operand[opc].predicate)
1208 (op[opc], mode))
d5a29419 1209 {
fca051af
AL		 1210 error ("%Kargument %d must be a constant immediate",
1211 exp, opc + 1 - have_retval);
d5a29419
KT		 1212 return const0_rtx;
1213 }
43e9d192
IB		 1214 break;
1215
1216 case SIMD_ARG_STOP:
1217 gcc_unreachable ();
1218 }
1219
d9e80f49 1220 opc++;
43e9d192
IB		 1221 }
1222 }
1223
d9e80f49
AL		 1224 switch (opc)
1225 {
1226 case 1:
1227 pat = GEN_FCN (icode) (op[0]);
1228 break;
43e9d192 1229
d9e80f49
AL		 1230 case 2:
1231 pat = GEN_FCN (icode) (op[0], op[1]);
1232 break;
43e9d192 1233
d9e80f49
AL		 1234 case 3:
1235 pat = GEN_FCN (icode) (op[0], op[1], op[2]);
1236 break;
43e9d192 1237
d9e80f49
AL		 1238 case 4:
1239 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
1240 break;
43e9d192 1241
d9e80f49
AL		 1242 case 5:
1243 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]);
1244 break;
43e9d192 1245
d9e80f49
AL		 1246 case 6:
1247 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]);
1248 break;
43e9d192 1249
d9e80f49
AL		 1250 default:
1251 gcc_unreachable ();
1252 }
43e9d192
IB		 1253
1254 if (!pat)
d5a29419 1255 return NULL_RTX;
43e9d192
IB		 1256
1257 emit_insn (pat);
1258
1259 return target;
1260}
1261
1262/* Expand an AArch64 AdvSIMD builtin(intrinsic). */
1263rtx
1264aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
1265{
661fce82
AL		 1266 if (fcode == AARCH64_SIMD_BUILTIN_LANE_CHECK)
1267 {
9c4f25cc
AP		 1268 rtx totalsize = expand_normal (CALL_EXPR_ARG (exp, 0));
1269 rtx elementsize = expand_normal (CALL_EXPR_ARG (exp, 1));
1270 if (CONST_INT_P (totalsize) && CONST_INT_P (elementsize)
1271 && UINTVAL (elementsize) != 0
1272 && UINTVAL (totalsize) != 0)
1273 {
1274 rtx lane_idx = expand_normal (CALL_EXPR_ARG (exp, 2));
1275 if (CONST_INT_P (lane_idx))
1276 aarch64_simd_lane_bounds (lane_idx, 0,
1277 UINTVAL (totalsize)
1278 / UINTVAL (elementsize),
1279 exp);
1280 else
1281 error ("%Klane index must be a constant immediate", exp);
1282 }
661fce82 1283 else
9c4f25cc 1284 error ("%Ktotal size and element size must be a non-zero constant immediate", exp);
661fce82
AL		 1285 /* Don't generate any RTL. */
1286 return const0_rtx;
1287 }
342be7f7 1288 aarch64_simd_builtin_datum *d =
661fce82 1289 &aarch64_simd_builtin_data[fcode - AARCH64_SIMD_PATTERN_START];
342be7f7 1290 enum insn_code icode = d->code;
0ff2bf46 1291 builtin_simd_arg args[SIMD_MAX_BUILTIN_ARGS + 1];
b5828b4b
JG		 1292 int num_args = insn_data[d->code].n_operands;
1293 int is_void = 0;
1294 int k;
43e9d192 1295
b5828b4b 1296 is_void = !!(d->qualifiers[0] & qualifier_void);
43e9d192 1297
b5828b4b
JG		 1298 num_args += is_void;
1299
1300 for (k = 1; k < num_args; k++)
1301 {
1302 /* We have four arrays of data, each indexed in a different fashion.
1303 qualifiers - element 0 always describes the function return type.
1304 operands - element 0 is either the operand for return value (if
1305 the function has a non-void return type) or the operand for the
1306 first argument.
1307 expr_args - element 0 always holds the first argument.
1308 args - element 0 is always used for the return type. */
1309 int qualifiers_k = k;
1310 int operands_k = k - is_void;
1311 int expr_args_k = k - 1;
1312
2a49c16d
AL		 1313 if (d->qualifiers[qualifiers_k] & qualifier_lane_index)
1314 args[k] = SIMD_ARG_LANE_INDEX;
9d63f43b
TC		 1315 else if (d->qualifiers[qualifiers_k] & qualifier_lane_pair_index)
1316 args[k] = SIMD_ARG_LANE_PAIR_INDEX;
4d0a0237
CB		 1317 else if (d->qualifiers[qualifiers_k] & qualifier_struct_load_store_lane_index)
1318 args[k] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX;
2a49c16d 1319 else if (d->qualifiers[qualifiers_k] & qualifier_immediate)
b5828b4b
JG		 1320 args[k] = SIMD_ARG_CONSTANT;
1321 else if (d->qualifiers[qualifiers_k] & qualifier_maybe_immediate)
1322 {
1323 rtx arg
1324 = expand_normal (CALL_EXPR_ARG (exp,
1325 (expr_args_k)));
1326 /* Handle constants only if the predicate allows it. */
1327 bool op_const_int_p =
1328 (CONST_INT_P (arg)
1329 && (*insn_data[icode].operand[operands_k].predicate)
1330 (arg, insn_data[icode].operand[operands_k].mode));
1331 args[k] = op_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG;
1332 }
1333 else
1334 args[k] = SIMD_ARG_COPY_TO_REG;
43e9d192 1335
43e9d192 1336 }
b5828b4b
JG		 1337 args[k] = SIMD_ARG_STOP;
1338
1339 /* The interface to aarch64_simd_expand_args expects a 0 if
1340 the function is void, and a 1 if it is not. */
1341 return aarch64_simd_expand_args
4d0a0237 1342 (target, icode, !is_void, exp, &args[1], d->mode);
43e9d192 1343}
342be7f7 1344
5d357f26
KT		 1345rtx
1346aarch64_crc32_expand_builtin (int fcode, tree exp, rtx target)
1347{
1348 rtx pat;
1349 aarch64_crc_builtin_datum *d
1350 = &aarch64_crc_builtin_data[fcode - (AARCH64_CRC32_BUILTIN_BASE + 1)];
1351 enum insn_code icode = d->icode;
1352 tree arg0 = CALL_EXPR_ARG (exp, 0);
1353 tree arg1 = CALL_EXPR_ARG (exp, 1);
1354 rtx op0 = expand_normal (arg0);
1355 rtx op1 = expand_normal (arg1);
ef4bddc2
RS		 1356 machine_mode tmode = insn_data[icode].operand[0].mode;
1357 machine_mode mode0 = insn_data[icode].operand[1].mode;
1358 machine_mode mode1 = insn_data[icode].operand[2].mode;
5d357f26
KT		 1359
1360 if (! target
1361 || GET_MODE (target) != tmode
1362 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
1363 target = gen_reg_rtx (tmode);
1364
1365 gcc_assert ((GET_MODE (op0) == mode0 || GET_MODE (op0) == VOIDmode)
1366 && (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode));
1367
1368 if (! (*insn_data[icode].operand[1].predicate) (op0, mode0))
1369 op0 = copy_to_mode_reg (mode0, op0);
1370 if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
1371 op1 = copy_to_mode_reg (mode1, op1);
1372
1373 pat = GEN_FCN (icode) (target, op0, op1);
d5a29419
KT		 1374 if (!pat)
1375 return NULL_RTX;
1376
5d357f26
KT		 1377 emit_insn (pat);
1378 return target;
1379}
1380
a6fc00da
BH		 1381/* Function to expand reciprocal square root builtins. */
1382
1383static rtx
1384aarch64_expand_builtin_rsqrt (int fcode, tree exp, rtx target)
1385{
1386 tree arg0 = CALL_EXPR_ARG (exp, 0);
1387 rtx op0 = expand_normal (arg0);
1388
1389 rtx (*gen) (rtx, rtx);
1390
1391 switch (fcode)
1392 {
1393 case AARCH64_BUILTIN_RSQRT_DF:
ee62a5a6 1394 gen = gen_rsqrtdf2;
a6fc00da
BH		 1395 break;
1396 case AARCH64_BUILTIN_RSQRT_SF:
ee62a5a6 1397 gen = gen_rsqrtsf2;
a6fc00da
BH		 1398 break;
1399 case AARCH64_BUILTIN_RSQRT_V2DF:
ee62a5a6 1400 gen = gen_rsqrtv2df2;
a6fc00da
BH		 1401 break;
1402 case AARCH64_BUILTIN_RSQRT_V2SF:
ee62a5a6 1403 gen = gen_rsqrtv2sf2;
a6fc00da
BH		 1404 break;
1405 case AARCH64_BUILTIN_RSQRT_V4SF:
ee62a5a6 1406 gen = gen_rsqrtv4sf2;
a6fc00da
BH		 1407 break;
1408 default: gcc_unreachable ();
1409 }
1410
1411 if (!target)
1412 target = gen_reg_rtx (GET_MODE (op0));
1413
1414 emit_insn (gen (target, op0));
1415
1416 return target;
1417}
1418
9d63f43b
TC		 1419/* Expand a FCMLA lane expression EXP with code FCODE and
1420 result going to TARGET if that is convenient. */
1421
1422rtx
1423aarch64_expand_fcmla_builtin (tree exp, rtx target, int fcode)
1424{
1425 int bcode = fcode - AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE - 1;
1426 aarch64_fcmla_laneq_builtin_datum* d
1427 = &aarch64_fcmla_lane_builtin_data[bcode];
1428 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
1429 rtx op0 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 0)));
1430 rtx op1 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 1)));
1431 rtx op2 = force_reg (quadmode, expand_normal (CALL_EXPR_ARG (exp, 2)));
1432 tree tmp = CALL_EXPR_ARG (exp, 3);
1433 rtx lane_idx = expand_expr (tmp, NULL_RTX, VOIDmode, EXPAND_INITIALIZER);
1434
1435 /* Validate that the lane index is a constant. */
1436 if (!CONST_INT_P (lane_idx))
1437 {
1438 error ("%Kargument %d must be a constant immediate", exp, 4);
1439 return const0_rtx;
1440 }
1441
1442 /* Validate that the index is within the expected range. */
1443 int nunits = GET_MODE_NUNITS (quadmode).to_constant ();
1444 aarch64_simd_lane_bounds (lane_idx, 0, nunits / 2, exp);
1445
1446 /* Keep to GCC-vector-extension lane indices in the RTL. */
1447 lane_idx = aarch64_endian_lane_rtx (quadmode, INTVAL (lane_idx));
1448
1449 /* Generate the correct register and mode. */
1450 int lane = INTVAL (lane_idx);
1451
1452 if (lane < nunits / 4)
1453 op2 = simplify_gen_subreg (d->mode, op2, quadmode, 0);
1454 else
1455 {
1456 /* Select the upper 64 bits, either a V2SF or V4HF, this however
1457 is quite messy, as the operation required even though simple
1458 doesn't have a simple RTL pattern, and seems it's quite hard to
1459 define using a single RTL pattern. The target generic version
1460 gen_highpart_mode generates code that isn't optimal. */
1461 rtx temp1 = gen_reg_rtx (d->mode);
1462 rtx temp2 = gen_reg_rtx (DImode);
1463 temp1 = simplify_gen_subreg (d->mode, op2, quadmode, 0);
1464 temp1 = simplify_gen_subreg (V2DImode, temp1, d->mode, 0);
1465 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1 , const1_rtx));
1466 op2 = simplify_gen_subreg (d->mode, temp2, GET_MODE (temp2), 0);
1467
1468 /* And recalculate the index. */
1469 lane -= nunits / 4;
1470 }
1471
1472 if (!target)
1473 target = gen_reg_rtx (d->mode);
1474 else
1475 target = force_reg (d->mode, target);
1476
1477 rtx pat = NULL_RTX;
1478
1479 if (d->lane)
1480 pat = GEN_FCN (d->icode) (target, op0, op1, op2,
1481 gen_int_mode (lane, SImode));
1482 else
1483 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
1484
1485 if (!pat)
1486 return NULL_RTX;
1487
1488 emit_insn (pat);
1489 return target;
1490}
1491
342be7f7
JG		 1492/* Expand an expression EXP that calls a built-in function,
1493 with result going to TARGET if that's convenient. */
1494rtx
1495aarch64_expand_builtin (tree exp,
1496 rtx target,
1497 rtx subtarget ATTRIBUTE_UNUSED,
ef4bddc2 1498 machine_mode mode ATTRIBUTE_UNUSED,
342be7f7
JG		 1499 int ignore ATTRIBUTE_UNUSED)
1500{
1501 tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
1502 int fcode = DECL_FUNCTION_CODE (fndecl);
aa87aced
KV		 1503 int icode;
1504 rtx pat, op0;
1505 tree arg0;
1506
1507 switch (fcode)
1508 {
1509 case AARCH64_BUILTIN_GET_FPCR:
1510 case AARCH64_BUILTIN_SET_FPCR:
1511 case AARCH64_BUILTIN_GET_FPSR:
1512 case AARCH64_BUILTIN_SET_FPSR:
1513 if ((fcode == AARCH64_BUILTIN_GET_FPCR)
1514 || (fcode == AARCH64_BUILTIN_GET_FPSR))
1515 {
1516 icode = (fcode == AARCH64_BUILTIN_GET_FPSR) ?
1517 CODE_FOR_get_fpsr : CODE_FOR_get_fpcr;
1518 target = gen_reg_rtx (SImode);
1519 pat = GEN_FCN (icode) (target);
1520 }
1521 else
1522 {
1523 target = NULL_RTX;
1524 icode = (fcode == AARCH64_BUILTIN_SET_FPSR) ?
1525 CODE_FOR_set_fpsr : CODE_FOR_set_fpcr;
1526 arg0 = CALL_EXPR_ARG (exp, 0);
e6cf8d65 1527 op0 = force_reg (SImode, expand_normal (arg0));
aa87aced
KV		 1528 pat = GEN_FCN (icode) (op0);
1529 }
1530 emit_insn (pat);
1531 return target;
312492bd
JW		 1532
1533 case AARCH64_PAUTH_BUILTIN_AUTIA1716:
1534 case AARCH64_PAUTH_BUILTIN_PACIA1716:
1535 case AARCH64_PAUTH_BUILTIN_XPACLRI:
1536 arg0 = CALL_EXPR_ARG (exp, 0);
1537 op0 = force_reg (Pmode, expand_normal (arg0));
1538
1539 if (!target)
1540 target = gen_reg_rtx (Pmode);
1541 else
1542 target = force_reg (Pmode, target);
1543
1544 emit_move_insn (target, op0);
1545
1546 if (fcode == AARCH64_PAUTH_BUILTIN_XPACLRI)
1547 {
1548 rtx lr = gen_rtx_REG (Pmode, R30_REGNUM);
1549 icode = CODE_FOR_xpaclri;
1550 emit_move_insn (lr, op0);
1551 emit_insn (GEN_FCN (icode) ());
1552 emit_move_insn (target, lr);
1553 }
1554 else
1555 {
1556 tree arg1 = CALL_EXPR_ARG (exp, 1);
1557 rtx op1 = force_reg (Pmode, expand_normal (arg1));
1558 icode = (fcode == AARCH64_PAUTH_BUILTIN_PACIA1716
1559 ? CODE_FOR_paci1716 : CODE_FOR_auti1716);
1560
1561 rtx x16_reg = gen_rtx_REG (Pmode, R16_REGNUM);
1562 rtx x17_reg = gen_rtx_REG (Pmode, R17_REGNUM);
1563 emit_move_insn (x17_reg, op0);
1564 emit_move_insn (x16_reg, op1);
1565 emit_insn (GEN_FCN (icode) ());
1566 emit_move_insn (target, x17_reg);
1567 }
1568
1569 return target;
9d63f43b
TC		 1570
1571 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF:
1572 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF:
1573 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF:
1574 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF:
1575 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF:
1576 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF:
1577 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF:
1578 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF:
1579 return aarch64_expand_fcmla_builtin (exp, target, fcode);
aa87aced 1580 }
342be7f7 1581
5d357f26 1582 if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX)
342be7f7 1583 return aarch64_simd_expand_builtin (fcode, exp, target);
5d357f26
KT		 1584 else if (fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX)
1585 return aarch64_crc32_expand_builtin (fcode, exp, target);
342be7f7 1586
a6fc00da
BH		 1587 if (fcode == AARCH64_BUILTIN_RSQRT_DF
1588 || fcode == AARCH64_BUILTIN_RSQRT_SF
1589 || fcode == AARCH64_BUILTIN_RSQRT_V2DF
1590 || fcode == AARCH64_BUILTIN_RSQRT_V2SF
1591 || fcode == AARCH64_BUILTIN_RSQRT_V4SF)
1592 return aarch64_expand_builtin_rsqrt (fcode, exp, target);
1593
d5a29419 1594 gcc_unreachable ();
342be7f7 1595}
42fc9a7f
JG		 1596
1597tree
10766209
RS		 1598aarch64_builtin_vectorized_function (unsigned int fn, tree type_out,
1599 tree type_in)
42fc9a7f 1600{
ef4bddc2 1601 machine_mode in_mode, out_mode;
6a70badb 1602 unsigned HOST_WIDE_INT in_n, out_n;
42fc9a7f
JG		 1603
1604 if (TREE_CODE (type_out) != VECTOR_TYPE
1605 || TREE_CODE (type_in) != VECTOR_TYPE)
1606 return NULL_TREE;
1607
1608 out_mode = TYPE_MODE (TREE_TYPE (type_out));
42fc9a7f 1609 in_mode = TYPE_MODE (TREE_TYPE (type_in));
6a70badb
RS		 1610 if (!TYPE_VECTOR_SUBPARTS (type_out).is_constant (&out_n)
1611 || !TYPE_VECTOR_SUBPARTS (type_in).is_constant (&in_n))
1612 return NULL_TREE;
42fc9a7f
JG		 1613
1614#undef AARCH64_CHECK_BUILTIN_MODE
1615#define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
1616#define AARCH64_FIND_FRINT_VARIANT(N) \
1617 (AARCH64_CHECK_BUILTIN_MODE (2, D) \
e993fea1 1618 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \
42fc9a7f 1619 : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
e993fea1 1620 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \
42fc9a7f 1621 : (AARCH64_CHECK_BUILTIN_MODE (2, S) \
e993fea1 1622 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \
42fc9a7f 1623 : NULL_TREE)))
10766209 1624 switch (fn)
42fc9a7f 1625 {
42fc9a7f
JG		 1626#undef AARCH64_CHECK_BUILTIN_MODE
1627#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1628 (out_mode == N##Fmode && out_n == C \
1629 && in_mode == N##Fmode && in_n == C)
10766209
RS		 1630 CASE_CFN_FLOOR:
1631 return AARCH64_FIND_FRINT_VARIANT (floor);
1632 CASE_CFN_CEIL:
1633 return AARCH64_FIND_FRINT_VARIANT (ceil);
1634 CASE_CFN_TRUNC:
1635 return AARCH64_FIND_FRINT_VARIANT (btrunc);
1636 CASE_CFN_ROUND:
1637 return AARCH64_FIND_FRINT_VARIANT (round);
1638 CASE_CFN_NEARBYINT:
1639 return AARCH64_FIND_FRINT_VARIANT (nearbyint);
1640 CASE_CFN_SQRT:
1641 return AARCH64_FIND_FRINT_VARIANT (sqrt);
42fc9a7f 1642#undef AARCH64_CHECK_BUILTIN_MODE
b5574232
VP		 1643#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1644 (out_mode == SImode && out_n == C \
1645 && in_mode == N##Imode && in_n == C)
10766209
RS		 1646 CASE_CFN_CLZ:
1647 {
1648 if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1649 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_clzv4si];
1650 return NULL_TREE;
1651 }
1652 CASE_CFN_CTZ:
1653 {
1654 if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1655 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si];
1656 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1657 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si];
1658 return NULL_TREE;
1659 }
b5574232 1660#undef AARCH64_CHECK_BUILTIN_MODE
42fc9a7f
JG		 1661#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1662 (out_mode == N##Imode && out_n == C \
1663 && in_mode == N##Fmode && in_n == C)
10766209
RS		 1664 CASE_CFN_IFLOOR:
1665 CASE_CFN_LFLOOR:
1666 CASE_CFN_LLFLOOR:
1667 {
1668 enum aarch64_builtins builtin;
1669 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1670 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di;
1671 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1672 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si;
1673 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1674 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si;
1675 else
1676 return NULL_TREE;
1677
1678 return aarch64_builtin_decls[builtin];
1679 }
1680 CASE_CFN_ICEIL:
1681 CASE_CFN_LCEIL:
1682 CASE_CFN_LLCEIL:
1683 {
1684 enum aarch64_builtins builtin;
1685 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1686 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di;
1687 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1688 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si;
1689 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1690 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si;
1691 else
1692 return NULL_TREE;
1693
1694 return aarch64_builtin_decls[builtin];
1695 }
1696 CASE_CFN_IROUND:
1697 CASE_CFN_LROUND:
1698 CASE_CFN_LLROUND:
1699 {
1700 enum aarch64_builtins builtin;
1701 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1702 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di;
1703 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1704 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si;
1705 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1706 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si;
1707 else
1708 return NULL_TREE;
1709
1710 return aarch64_builtin_decls[builtin];
1711 }
1712 case CFN_BUILT_IN_BSWAP16:
c7f28cd5
KT		 1713#undef AARCH64_CHECK_BUILTIN_MODE
1714#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1715 (out_mode == N##Imode && out_n == C \
1716 && in_mode == N##Imode && in_n == C)
10766209
RS		 1717 if (AARCH64_CHECK_BUILTIN_MODE (4, H))
1718 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4hi];
1719 else if (AARCH64_CHECK_BUILTIN_MODE (8, H))
1720 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv8hi];
1721 else
1722 return NULL_TREE;
1723 case CFN_BUILT_IN_BSWAP32:
1724 if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1725 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2si];
1726 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1727 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4si];
1728 else
1729 return NULL_TREE;
1730 case CFN_BUILT_IN_BSWAP64:
1731 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1732 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2di];
1733 else
1734 return NULL_TREE;
1735 default:
1736 return NULL_TREE;
42fc9a7f
JG		 1737 }
1738
1739 return NULL_TREE;
1740}
0ac198d3 1741
a6fc00da
BH		 1742/* Return builtin for reciprocal square root. */
1743
1744tree
ee62a5a6 1745aarch64_builtin_rsqrt (unsigned int fn)
a6fc00da 1746{
ee62a5a6
RS		 1747 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df)
1748 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF];
1749 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf)
1750 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF];
1751 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf)
1752 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF];
a6fc00da
BH		 1753 return NULL_TREE;
1754}
1755
0ac198d3
JG		 1756#undef VAR1
1757#define VAR1(T, N, MAP, A) \
e993fea1 1758 case AARCH64_SIMD_BUILTIN_##T##_##N##A:
0ac198d3 1759
9697e620
JG		 1760tree
1761aarch64_fold_builtin (tree fndecl, int n_args ATTRIBUTE_UNUSED, tree *args,
1762 bool ignore ATTRIBUTE_UNUSED)
1763{
1764 int fcode = DECL_FUNCTION_CODE (fndecl);
1765 tree type = TREE_TYPE (TREE_TYPE (fndecl));
1766
1767 switch (fcode)
1768 {
8f905d69 1769 BUILTIN_VDQF (UNOP, abs, 2)
9697e620 1770 return fold_build1 (ABS_EXPR, type, args[0]);
1709ff9b
JG		 1771 VAR1 (UNOP, floatv2si, 2, v2sf)
1772 VAR1 (UNOP, floatv4si, 2, v4sf)
1773 VAR1 (UNOP, floatv2di, 2, v2df)
1774 return fold_build1 (FLOAT_EXPR, type, args[0]);
9697e620
JG		 1775 default:
1776 break;
1777 }
1778
1779 return NULL_TREE;
1780}
1781
0ac198d3
JG		 1782bool
1783aarch64_gimple_fold_builtin (gimple_stmt_iterator *gsi)
1784{
1785 bool changed = false;
355fe088 1786 gimple *stmt = gsi_stmt (*gsi);
0ac198d3
JG		 1787 tree call = gimple_call_fn (stmt);
1788 tree fndecl;
355fe088 1789 gimple *new_stmt = NULL;
22756ccf 1790
0ac198d3
JG		 1791 if (call)
1792 {
1793 fndecl = gimple_call_fndecl (stmt);
1794 if (fndecl)
1795 {
1796 int fcode = DECL_FUNCTION_CODE (fndecl);
546e500c 1797 unsigned nargs = gimple_call_num_args (stmt);
0ac198d3
JG		 1798 tree *args = (nargs > 0
1799 ? gimple_call_arg_ptr (stmt, 0)
1800 : &error_mark_node);
1801
16d24520 1802 /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int
fc72cba7
AL		 1803 and unsigned int; it will distinguish according to the types of
1804 the arguments to the __builtin. */
0ac198d3
JG		 1805 switch (fcode)
1806 {
fc72cba7 1807 BUILTIN_VALL (UNOP, reduc_plus_scal_, 10)
16d24520
RS		 1808 new_stmt = gimple_build_call_internal (IFN_REDUC_PLUS,
1809 1, args[0]);
1810 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
0ac198d3 1811 break;
fc72cba7
AL		 1812 BUILTIN_VDQIF (UNOP, reduc_smax_scal_, 10)
1813 BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10)
16d24520
RS		 1814 new_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
1815 1, args[0]);
1816 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
1598945b 1817 break;
fc72cba7
AL		 1818 BUILTIN_VDQIF (UNOP, reduc_smin_scal_, 10)
1819 BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10)
16d24520
RS		 1820 new_stmt = gimple_build_call_internal (IFN_REDUC_MIN,
1821 1, args[0]);
1822 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
1598945b 1823 break;
546e500c
BB		 1824 BUILTIN_GPF (BINOP, fmulx, 0)
1825 {
1826 gcc_assert (nargs == 2);
1827 bool a0_cst_p = TREE_CODE (args[0]) == REAL_CST;
1828 bool a1_cst_p = TREE_CODE (args[1]) == REAL_CST;
1829 if (a0_cst_p || a1_cst_p)
1830 {
1831 if (a0_cst_p && a1_cst_p)
1832 {
1833 tree t0 = TREE_TYPE (args[0]);
1834 real_value a0 = (TREE_REAL_CST (args[0]));
1835 real_value a1 = (TREE_REAL_CST (args[1]));
1836 if (real_equal (&a1, &dconst0))
1837 std::swap (a0, a1);
1838 /* According to real_equal (), +0 equals -0. */
1839 if (real_equal (&a0, &dconst0) && real_isinf (&a1))
1840 {
1841 real_value res = dconst2;
1842 res.sign = a0.sign ^ a1.sign;
1843 new_stmt =
1844 gimple_build_assign (gimple_call_lhs (stmt),
1845 REAL_CST,
1846 build_real (t0, res));
1847 }
1848 else
1849 new_stmt =
1850 gimple_build_assign (gimple_call_lhs (stmt),
1851 MULT_EXPR,
1852 args[0], args[1]);
1853 }
1854 else /* a0_cst_p ^ a1_cst_p. */
1855 {
1856 real_value const_part = a0_cst_p
1857 ? TREE_REAL_CST (args[0]) : TREE_REAL_CST (args[1]);
1858 if (!real_equal (&const_part, &dconst0)
1859 && !real_isinf (&const_part))
1860 new_stmt =
1861 gimple_build_assign (gimple_call_lhs (stmt),
1862 MULT_EXPR, args[0], args[1]);
1863 }
1864 }
1865 if (new_stmt)
1866 {
1867 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
1868 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
1869 }
1870 break;
1871 }
0ac198d3
JG		 1872 default:
1873 break;
1874 }
1875 }
1876 }
1877
1878 if (new_stmt)
1879 {
1880 gsi_replace (gsi, new_stmt, true);
1881 changed = true;
1882 }
1883
1884 return changed;
1885}
1886
aa87aced
KV		 1887void
1888aarch64_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
1889{
1890 const unsigned AARCH64_FE_INVALID = 1;
1891 const unsigned AARCH64_FE_DIVBYZERO = 2;
1892 const unsigned AARCH64_FE_OVERFLOW = 4;
1893 const unsigned AARCH64_FE_UNDERFLOW = 8;
1894 const unsigned AARCH64_FE_INEXACT = 16;
1895 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT = (AARCH64_FE_INVALID
1896 | AARCH64_FE_DIVBYZERO
1897 | AARCH64_FE_OVERFLOW
1898 | AARCH64_FE_UNDERFLOW
1899 | AARCH64_FE_INEXACT);
1900 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT = 8;
1901 tree fenv_cr, fenv_sr, get_fpcr, set_fpcr, mask_cr, mask_sr;
1902 tree ld_fenv_cr, ld_fenv_sr, masked_fenv_cr, masked_fenv_sr, hold_fnclex_cr;
1903 tree hold_fnclex_sr, new_fenv_var, reload_fenv, restore_fnenv, get_fpsr, set_fpsr;
1904 tree update_call, atomic_feraiseexcept, hold_fnclex, masked_fenv, ld_fenv;
1905
1906 /* Generate the equivalence of :
1907 unsigned int fenv_cr;
1908 fenv_cr = __builtin_aarch64_get_fpcr ();
1909
1910 unsigned int fenv_sr;
1911 fenv_sr = __builtin_aarch64_get_fpsr ();
1912
1913 Now set all exceptions to non-stop
1914 unsigned int mask_cr
1915 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
1916 unsigned int masked_cr;
1917 masked_cr = fenv_cr & mask_cr;
1918
1919 And clear all exception flags
1920 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
1921 unsigned int masked_cr;
1922 masked_sr = fenv_sr & mask_sr;
1923
1924 __builtin_aarch64_set_cr (masked_cr);
1925 __builtin_aarch64_set_sr (masked_sr); */
1926
09ba9ef7
RR		 1927 fenv_cr = create_tmp_var_raw (unsigned_type_node);
1928 fenv_sr = create_tmp_var_raw (unsigned_type_node);
aa87aced
KV		 1929
1930 get_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR];
1931 set_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR];
1932 get_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR];
1933 set_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR];
1934
1935 mask_cr = build_int_cst (unsigned_type_node,
1936 ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT));
1937 mask_sr = build_int_cst (unsigned_type_node,
1938 ~(AARCH64_FE_ALL_EXCEPT));
1939
1940 ld_fenv_cr = build2 (MODIFY_EXPR, unsigned_type_node,
09ba9ef7 1941 fenv_cr, build_call_expr (get_fpcr, 0));
aa87aced 1942 ld_fenv_sr = build2 (MODIFY_EXPR, unsigned_type_node,
09ba9ef7 1943 fenv_sr, build_call_expr (get_fpsr, 0));
aa87aced
KV		 1944
1945 masked_fenv_cr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_cr, mask_cr);
1946 masked_fenv_sr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_sr, mask_sr);
1947
1948 hold_fnclex_cr = build_call_expr (set_fpcr, 1, masked_fenv_cr);
1949 hold_fnclex_sr = build_call_expr (set_fpsr, 1, masked_fenv_sr);
1950
1951 hold_fnclex = build2 (COMPOUND_EXPR, void_type_node, hold_fnclex_cr,
1952 hold_fnclex_sr);
1953 masked_fenv = build2 (COMPOUND_EXPR, void_type_node, masked_fenv_cr,
1954 masked_fenv_sr);
1955 ld_fenv = build2 (COMPOUND_EXPR, void_type_node, ld_fenv_cr, ld_fenv_sr);
1956
1957 *hold = build2 (COMPOUND_EXPR, void_type_node,
1958 build2 (COMPOUND_EXPR, void_type_node, masked_fenv, ld_fenv),
1959 hold_fnclex);
1960
1961 /* Store the value of masked_fenv to clear the exceptions:
1962 __builtin_aarch64_set_fpsr (masked_fenv_sr); */
1963
1964 *clear = build_call_expr (set_fpsr, 1, masked_fenv_sr);
1965
1966 /* Generate the equivalent of :
1967 unsigned int new_fenv_var;
1968 new_fenv_var = __builtin_aarch64_get_fpsr ();
1969
1970 __builtin_aarch64_set_fpsr (fenv_sr);
1971
1972 __atomic_feraiseexcept (new_fenv_var); */
1973
09ba9ef7 1974 new_fenv_var = create_tmp_var_raw (unsigned_type_node);
aa87aced
KV		 1975 reload_fenv = build2 (MODIFY_EXPR, unsigned_type_node,
1976 new_fenv_var, build_call_expr (get_fpsr, 0));
1977 restore_fnenv = build_call_expr (set_fpsr, 1, fenv_sr);
1978 atomic_feraiseexcept = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
1979 update_call = build_call_expr (atomic_feraiseexcept, 1,
1980 fold_convert (integer_type_node, new_fenv_var));
1981 *update = build2 (COMPOUND_EXPR, void_type_node,
1982 build2 (COMPOUND_EXPR, void_type_node,
1983 reload_fenv, restore_fnenv), update_call);
1984}
1985
1986
42fc9a7f
JG		 1987#undef AARCH64_CHECK_BUILTIN_MODE
1988#undef AARCH64_FIND_FRINT_VARIANT
0ddec79f
JG		 1989#undef CF0
1990#undef CF1
1991#undef CF2
1992#undef CF3
1993#undef CF4
1994#undef CF10
1995#undef VAR1
1996#undef VAR2
1997#undef VAR3
1998#undef VAR4
1999#undef VAR5
2000#undef VAR6
2001#undef VAR7
2002#undef VAR8
2003#undef VAR9
2004#undef VAR10
2005#undef VAR11
2006
3c03d39d 2007#include "gt-aarch64-builtins.h"
Mirror of https://gcc.gnu.org/git/gcc.git