1 /* Builtins' description for AArch64 SIMD architecture.
2 Copyright (C) 2011-2023 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
25 #include "coretypes.h"
28 #include "basic-block.h"
38 #include "diagnostic-core.h"
39 #include "fold-const.h"
40 #include "stor-layout.h"
43 #include "langhooks.h"
44 #include "gimple-iterator.h"
45 #include "case-cfn-macros.h"
47 #include "stringpool.h"
49 #include "gimple-fold.h"
51 #define v8qi_UP E_V8QImode
52 #define v8di_UP E_V8DImode
53 #define v4hi_UP E_V4HImode
54 #define v4hf_UP E_V4HFmode
55 #define v2si_UP E_V2SImode
56 #define v2sf_UP E_V2SFmode
57 #define v1df_UP E_V1DFmode
58 #define v1di_UP E_V1DImode
59 #define di_UP E_DImode
60 #define df_UP E_DFmode
61 #define v16qi_UP E_V16QImode
62 #define v8hi_UP E_V8HImode
63 #define v8hf_UP E_V8HFmode
64 #define v4si_UP E_V4SImode
65 #define v4sf_UP E_V4SFmode
66 #define v2di_UP E_V2DImode
67 #define v2df_UP E_V2DFmode
68 #define ti_UP E_TImode
69 #define oi_UP E_OImode
70 #define ci_UP E_CImode
71 #define xi_UP E_XImode
72 #define si_UP E_SImode
73 #define sf_UP E_SFmode
74 #define hi_UP E_HImode
75 #define hf_UP E_HFmode
76 #define qi_UP E_QImode
77 #define bf_UP E_BFmode
78 #define v4bf_UP E_V4BFmode
79 #define v8bf_UP E_V8BFmode
80 #define v2x8qi_UP E_V2x8QImode
81 #define v2x4hi_UP E_V2x4HImode
82 #define v2x4hf_UP E_V2x4HFmode
83 #define v2x4bf_UP E_V2x4BFmode
84 #define v2x2si_UP E_V2x2SImode
85 #define v2x2sf_UP E_V2x2SFmode
86 #define v2x1di_UP E_V2x1DImode
87 #define v2x1df_UP E_V2x1DFmode
88 #define v2x16qi_UP E_V2x16QImode
89 #define v2x8hi_UP E_V2x8HImode
90 #define v2x8hf_UP E_V2x8HFmode
91 #define v2x8bf_UP E_V2x8BFmode
92 #define v2x4si_UP E_V2x4SImode
93 #define v2x4sf_UP E_V2x4SFmode
94 #define v2x2di_UP E_V2x2DImode
95 #define v2x2df_UP E_V2x2DFmode
96 #define v3x8qi_UP E_V3x8QImode
97 #define v3x4hi_UP E_V3x4HImode
98 #define v3x4hf_UP E_V3x4HFmode
99 #define v3x4bf_UP E_V3x4BFmode
100 #define v3x2si_UP E_V3x2SImode
101 #define v3x2sf_UP E_V3x2SFmode
102 #define v3x1di_UP E_V3x1DImode
103 #define v3x1df_UP E_V3x1DFmode
104 #define v3x16qi_UP E_V3x16QImode
105 #define v3x8hi_UP E_V3x8HImode
106 #define v3x8hf_UP E_V3x8HFmode
107 #define v3x8bf_UP E_V3x8BFmode
108 #define v3x4si_UP E_V3x4SImode
109 #define v3x4sf_UP E_V3x4SFmode
110 #define v3x2di_UP E_V3x2DImode
111 #define v3x2df_UP E_V3x2DFmode
112 #define v4x8qi_UP E_V4x8QImode
113 #define v4x4hi_UP E_V4x4HImode
114 #define v4x4hf_UP E_V4x4HFmode
115 #define v4x4bf_UP E_V4x4BFmode
116 #define v4x2si_UP E_V4x2SImode
117 #define v4x2sf_UP E_V4x2SFmode
118 #define v4x1di_UP E_V4x1DImode
119 #define v4x1df_UP E_V4x1DFmode
120 #define v4x16qi_UP E_V4x16QImode
121 #define v4x8hi_UP E_V4x8HImode
122 #define v4x8hf_UP E_V4x8HFmode
123 #define v4x8bf_UP E_V4x8BFmode
124 #define v4x4si_UP E_V4x4SImode
125 #define v4x4sf_UP E_V4x4SFmode
126 #define v4x2di_UP E_V4x2DImode
127 #define v4x2df_UP E_V4x2DFmode
130 #define MODE_d_bf16 E_V4BFmode
131 #define MODE_d_f16 E_V4HFmode
132 #define MODE_d_f32 E_V2SFmode
133 #define MODE_d_f64 E_V1DFmode
134 #define MODE_d_s8 E_V8QImode
135 #define MODE_d_s16 E_V4HImode
136 #define MODE_d_s32 E_V2SImode
137 #define MODE_d_s64 E_V1DImode
138 #define MODE_d_u8 E_V8QImode
139 #define MODE_d_u16 E_V4HImode
140 #define MODE_d_u32 E_V2SImode
141 #define MODE_d_u64 E_V1DImode
142 #define MODE_d_p8 E_V8QImode
143 #define MODE_d_p16 E_V4HImode
144 #define MODE_d_p64 E_V1DImode
145 #define MODE_q_bf16 E_V8BFmode
146 #define MODE_q_f16 E_V8HFmode
147 #define MODE_q_f32 E_V4SFmode
148 #define MODE_q_f64 E_V2DFmode
149 #define MODE_q_s8 E_V16QImode
150 #define MODE_q_s16 E_V8HImode
151 #define MODE_q_s32 E_V4SImode
152 #define MODE_q_s64 E_V2DImode
153 #define MODE_q_u8 E_V16QImode
154 #define MODE_q_u16 E_V8HImode
155 #define MODE_q_u32 E_V4SImode
156 #define MODE_q_u64 E_V2DImode
157 #define MODE_q_p8 E_V16QImode
158 #define MODE_q_p16 E_V8HImode
159 #define MODE_q_p64 E_V2DImode
160 #define MODE_q_p128 E_TImode
162 #define QUAL_bf16 qualifier_none
163 #define QUAL_f16 qualifier_none
164 #define QUAL_f32 qualifier_none
165 #define QUAL_f64 qualifier_none
166 #define QUAL_s8 qualifier_none
167 #define QUAL_s16 qualifier_none
168 #define QUAL_s32 qualifier_none
169 #define QUAL_s64 qualifier_none
170 #define QUAL_u8 qualifier_unsigned
171 #define QUAL_u16 qualifier_unsigned
172 #define QUAL_u32 qualifier_unsigned
173 #define QUAL_u64 qualifier_unsigned
174 #define QUAL_p8 qualifier_poly
175 #define QUAL_p16 qualifier_poly
176 #define QUAL_p64 qualifier_poly
177 #define QUAL_p128 qualifier_poly
182 #define SIMD_INTR_MODE(suffix, length) MODE_##length##_##suffix
183 #define SIMD_INTR_QUAL(suffix) QUAL_##suffix
184 #define SIMD_INTR_LENGTH_CHAR(length) LENGTH_##length
187 #define SIMD_MAX_BUILTIN_ARGS 5
189 enum aarch64_type_qualifiers
192 qualifier_none
= 0x0,
193 /* unsigned T foo. */
194 qualifier_unsigned
= 0x1, /* 1 << 0 */
196 qualifier_const
= 0x2, /* 1 << 1 */
198 qualifier_pointer
= 0x4, /* 1 << 2 */
199 /* Used when expanding arguments if an operand could
201 qualifier_immediate
= 0x8, /* 1 << 3 */
202 qualifier_maybe_immediate
= 0x10, /* 1 << 4 */
203 /* void foo (...). */
204 qualifier_void
= 0x20, /* 1 << 5 */
205 /* 1 << 6 is now unused */
206 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
207 rather than using the type of the operand. */
208 qualifier_map_mode
= 0x80, /* 1 << 7 */
209 /* qualifier_pointer | qualifier_map_mode */
210 qualifier_pointer_map_mode
= 0x84,
211 /* qualifier_const | qualifier_pointer | qualifier_map_mode */
212 qualifier_const_pointer_map_mode
= 0x86,
213 /* Polynomial types. */
214 qualifier_poly
= 0x100,
215 /* Lane indices - must be in range, and flipped for bigendian. */
216 qualifier_lane_index
= 0x200,
217 /* Lane indices for single lane structure loads and stores. */
218 qualifier_struct_load_store_lane_index
= 0x400,
219 /* Lane indices selected in pairs. - must be in range, and flipped for
221 qualifier_lane_pair_index
= 0x800,
222 /* Lane indices selected in quadtuplets. - must be in range, and flipped for
224 qualifier_lane_quadtup_index
= 0x1000,
227 /* Flags that describe what a function might do. */
228 const unsigned int FLAG_NONE
= 0U;
229 const unsigned int FLAG_READ_FPCR
= 1U << 0;
230 const unsigned int FLAG_RAISE_FP_EXCEPTIONS
= 1U << 1;
231 const unsigned int FLAG_READ_MEMORY
= 1U << 2;
232 const unsigned int FLAG_PREFETCH_MEMORY
= 1U << 3;
233 const unsigned int FLAG_WRITE_MEMORY
= 1U << 4;
235 /* Not all FP intrinsics raise FP exceptions or read FPCR register,
236 use this flag to suppress it. */
237 const unsigned int FLAG_AUTO_FP
= 1U << 5;
239 const unsigned int FLAG_FP
= FLAG_READ_FPCR
| FLAG_RAISE_FP_EXCEPTIONS
;
240 const unsigned int FLAG_ALL
= FLAG_READ_FPCR
| FLAG_RAISE_FP_EXCEPTIONS
241 | FLAG_READ_MEMORY
| FLAG_PREFETCH_MEMORY
| FLAG_WRITE_MEMORY
;
242 const unsigned int FLAG_STORE
= FLAG_WRITE_MEMORY
| FLAG_AUTO_FP
;
243 const unsigned int FLAG_LOAD
= FLAG_READ_MEMORY
| FLAG_AUTO_FP
;
249 const enum insn_code code
;
251 enum aarch64_type_qualifiers
*qualifiers
;
253 } aarch64_simd_builtin_datum
;
255 static enum aarch64_type_qualifiers
256 aarch64_types_unop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
257 = { qualifier_none
, qualifier_none
};
258 #define TYPES_UNOP (aarch64_types_unop_qualifiers)
259 static enum aarch64_type_qualifiers
260 aarch64_types_unopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
261 = { qualifier_unsigned
, qualifier_unsigned
};
262 #define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
263 static enum aarch64_type_qualifiers
264 aarch64_types_unopus_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
265 = { qualifier_unsigned
, qualifier_none
};
266 #define TYPES_UNOPUS (aarch64_types_unopus_qualifiers)
267 static enum aarch64_type_qualifiers
268 aarch64_types_binop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
269 = { qualifier_none
, qualifier_none
, qualifier_maybe_immediate
};
270 #define TYPES_BINOP (aarch64_types_binop_qualifiers)
271 static enum aarch64_type_qualifiers
272 aarch64_types_binopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
273 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
};
274 #define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
275 static enum aarch64_type_qualifiers
276 aarch64_types_binop_uus_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
277 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_none
};
278 #define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
279 static enum aarch64_type_qualifiers
280 aarch64_types_binop_ssu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
281 = { qualifier_none
, qualifier_none
, qualifier_unsigned
};
282 #define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
283 static enum aarch64_type_qualifiers
284 aarch64_types_binop_uss_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
285 = { qualifier_unsigned
, qualifier_none
, qualifier_none
};
286 #define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers)
287 static enum aarch64_type_qualifiers
288 aarch64_types_binopp_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
289 = { qualifier_poly
, qualifier_poly
, qualifier_poly
};
290 #define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
291 static enum aarch64_type_qualifiers
292 aarch64_types_binop_ppu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
293 = { qualifier_poly
, qualifier_poly
, qualifier_unsigned
};
294 #define TYPES_BINOP_PPU (aarch64_types_binop_ppu_qualifiers)
296 static enum aarch64_type_qualifiers
297 aarch64_types_ternop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
298 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_none
};
299 #define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
300 static enum aarch64_type_qualifiers
301 aarch64_types_ternop_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
302 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_lane_index
};
303 #define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
304 static enum aarch64_type_qualifiers
305 aarch64_types_ternopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
306 = { qualifier_unsigned
, qualifier_unsigned
,
307 qualifier_unsigned
, qualifier_unsigned
};
308 #define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
309 static enum aarch64_type_qualifiers
310 aarch64_types_ternopu_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
311 = { qualifier_unsigned
, qualifier_unsigned
,
312 qualifier_unsigned
, qualifier_lane_index
};
313 #define TYPES_TERNOPU_LANE (aarch64_types_ternopu_lane_qualifiers)
314 static enum aarch64_type_qualifiers
315 aarch64_types_ternopu_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
316 = { qualifier_unsigned
, qualifier_unsigned
,
317 qualifier_unsigned
, qualifier_immediate
};
318 #define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers)
319 static enum aarch64_type_qualifiers
320 aarch64_types_ternop_sssu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
321 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_unsigned
};
322 #define TYPES_TERNOP_SSSU (aarch64_types_ternop_sssu_qualifiers)
323 static enum aarch64_type_qualifiers
324 aarch64_types_ternop_ssus_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
325 = { qualifier_none
, qualifier_none
, qualifier_unsigned
, qualifier_none
};
326 #define TYPES_TERNOP_SSUS (aarch64_types_ternop_ssus_qualifiers)
327 static enum aarch64_type_qualifiers
328 aarch64_types_ternop_suss_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
329 = { qualifier_none
, qualifier_unsigned
, qualifier_none
, qualifier_none
};
330 #define TYPES_TERNOP_SUSS (aarch64_types_ternop_suss_qualifiers)
331 static enum aarch64_type_qualifiers
332 aarch64_types_binop_pppu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
333 = { qualifier_poly
, qualifier_poly
, qualifier_poly
, qualifier_unsigned
};
334 #define TYPES_TERNOP_PPPU (aarch64_types_binop_pppu_qualifiers)
336 static enum aarch64_type_qualifiers
337 aarch64_types_quadop_lane_pair_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
338 = { qualifier_none
, qualifier_none
, qualifier_none
,
339 qualifier_none
, qualifier_lane_pair_index
};
340 #define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers)
341 static enum aarch64_type_qualifiers
342 aarch64_types_quadop_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
343 = { qualifier_none
, qualifier_none
, qualifier_none
,
344 qualifier_none
, qualifier_lane_index
};
345 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
346 static enum aarch64_type_qualifiers
347 aarch64_types_quadopu_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
348 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
349 qualifier_unsigned
, qualifier_lane_index
};
350 #define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
352 static enum aarch64_type_qualifiers
353 aarch64_types_quadopssus_lane_quadtup_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
354 = { qualifier_none
, qualifier_none
, qualifier_unsigned
,
355 qualifier_none
, qualifier_lane_quadtup_index
};
356 #define TYPES_QUADOPSSUS_LANE_QUADTUP \
357 (aarch64_types_quadopssus_lane_quadtup_qualifiers)
358 static enum aarch64_type_qualifiers
359 aarch64_types_quadopsssu_lane_quadtup_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
360 = { qualifier_none
, qualifier_none
, qualifier_none
,
361 qualifier_unsigned
, qualifier_lane_quadtup_index
};
362 #define TYPES_QUADOPSSSU_LANE_QUADTUP \
363 (aarch64_types_quadopsssu_lane_quadtup_qualifiers)
365 static enum aarch64_type_qualifiers
366 aarch64_types_quadopu_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
367 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
368 qualifier_unsigned
, qualifier_immediate
};
369 #define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
371 static enum aarch64_type_qualifiers
372 aarch64_types_binop_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
373 = { qualifier_none
, qualifier_none
, qualifier_immediate
};
374 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
375 #define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
376 static enum aarch64_type_qualifiers
377 aarch64_types_shift_to_unsigned_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
378 = { qualifier_unsigned
, qualifier_none
, qualifier_immediate
};
379 #define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
380 static enum aarch64_type_qualifiers
381 aarch64_types_fcvt_from_unsigned_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
382 = { qualifier_none
, qualifier_unsigned
, qualifier_immediate
};
383 #define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers)
384 static enum aarch64_type_qualifiers
385 aarch64_types_unsigned_shift_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
386 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_immediate
};
387 #define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
388 #define TYPES_USHIFT2IMM (aarch64_types_ternopu_imm_qualifiers)
389 static enum aarch64_type_qualifiers
390 aarch64_types_shift2_to_unsigned_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
391 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_none
, qualifier_immediate
};
392 #define TYPES_SHIFT2IMM_UUSS (aarch64_types_shift2_to_unsigned_qualifiers)
394 static enum aarch64_type_qualifiers
395 aarch64_types_ternop_s_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
396 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_immediate
};
397 #define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
398 #define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers)
399 #define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers)
400 #define TYPES_SHIFT2IMM (aarch64_types_ternop_s_imm_qualifiers)
402 static enum aarch64_type_qualifiers
403 aarch64_types_ternop_p_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
404 = { qualifier_poly
, qualifier_poly
, qualifier_poly
, qualifier_immediate
};
405 #define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers)
407 static enum aarch64_type_qualifiers
408 aarch64_types_unsigned_shiftacc_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
409 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
410 qualifier_immediate
};
411 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
413 static enum aarch64_type_qualifiers
414 aarch64_types_load1_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
415 = { qualifier_none
, qualifier_const_pointer_map_mode
};
416 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
417 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
418 static enum aarch64_type_qualifiers
419 aarch64_types_load1_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
420 = { qualifier_unsigned
, qualifier_const_pointer_map_mode
};
421 #define TYPES_LOAD1_U (aarch64_types_load1_u_qualifiers)
422 #define TYPES_LOADSTRUCT_U (aarch64_types_load1_u_qualifiers)
423 static enum aarch64_type_qualifiers
424 aarch64_types_load1_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
425 = { qualifier_poly
, qualifier_const_pointer_map_mode
};
426 #define TYPES_LOAD1_P (aarch64_types_load1_p_qualifiers)
427 #define TYPES_LOADSTRUCT_P (aarch64_types_load1_p_qualifiers)
429 static enum aarch64_type_qualifiers
430 aarch64_types_loadstruct_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
431 = { qualifier_none
, qualifier_const_pointer_map_mode
,
432 qualifier_none
, qualifier_struct_load_store_lane_index
};
433 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
434 static enum aarch64_type_qualifiers
435 aarch64_types_loadstruct_lane_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
436 = { qualifier_unsigned
, qualifier_const_pointer_map_mode
,
437 qualifier_unsigned
, qualifier_struct_load_store_lane_index
};
438 #define TYPES_LOADSTRUCT_LANE_U (aarch64_types_loadstruct_lane_u_qualifiers)
439 static enum aarch64_type_qualifiers
440 aarch64_types_loadstruct_lane_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
441 = { qualifier_poly
, qualifier_const_pointer_map_mode
,
442 qualifier_poly
, qualifier_struct_load_store_lane_index
};
443 #define TYPES_LOADSTRUCT_LANE_P (aarch64_types_loadstruct_lane_p_qualifiers)
445 static enum aarch64_type_qualifiers
446 aarch64_types_bsl_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
447 = { qualifier_poly
, qualifier_unsigned
,
448 qualifier_poly
, qualifier_poly
};
449 #define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
450 static enum aarch64_type_qualifiers
451 aarch64_types_bsl_s_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
452 = { qualifier_none
, qualifier_unsigned
,
453 qualifier_none
, qualifier_none
};
454 #define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
455 static enum aarch64_type_qualifiers
456 aarch64_types_bsl_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
457 = { qualifier_unsigned
, qualifier_unsigned
,
458 qualifier_unsigned
, qualifier_unsigned
};
459 #define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
461 /* The first argument (return type) of a store should be void type,
462 which we represent with qualifier_void. Their first operand will be
463 a DImode pointer to the location to store to, so we must use
464 qualifier_map_mode | qualifier_pointer to build a pointer to the
465 element type of the vector. */
466 static enum aarch64_type_qualifiers
467 aarch64_types_store1_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
468 = { qualifier_void
, qualifier_pointer_map_mode
, qualifier_none
};
469 #define TYPES_STORE1 (aarch64_types_store1_qualifiers)
470 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
471 static enum aarch64_type_qualifiers
472 aarch64_types_store1_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
473 = { qualifier_void
, qualifier_pointer_map_mode
, qualifier_unsigned
};
474 #define TYPES_STORE1_U (aarch64_types_store1_u_qualifiers)
475 #define TYPES_STORESTRUCT_U (aarch64_types_store1_u_qualifiers)
476 static enum aarch64_type_qualifiers
477 aarch64_types_store1_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
478 = { qualifier_void
, qualifier_pointer_map_mode
, qualifier_poly
};
479 #define TYPES_STORE1_P (aarch64_types_store1_p_qualifiers)
480 #define TYPES_STORESTRUCT_P (aarch64_types_store1_p_qualifiers)
482 static enum aarch64_type_qualifiers
483 aarch64_types_storestruct_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
484 = { qualifier_void
, qualifier_pointer_map_mode
,
485 qualifier_none
, qualifier_struct_load_store_lane_index
};
486 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
487 static enum aarch64_type_qualifiers
488 aarch64_types_storestruct_lane_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
489 = { qualifier_void
, qualifier_pointer_map_mode
,
490 qualifier_unsigned
, qualifier_struct_load_store_lane_index
};
491 #define TYPES_STORESTRUCT_LANE_U (aarch64_types_storestruct_lane_u_qualifiers)
492 static enum aarch64_type_qualifiers
493 aarch64_types_storestruct_lane_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
494 = { qualifier_void
, qualifier_pointer_map_mode
,
495 qualifier_poly
, qualifier_struct_load_store_lane_index
};
496 #define TYPES_STORESTRUCT_LANE_P (aarch64_types_storestruct_lane_p_qualifiers)
498 #define CF0(N, X) CODE_FOR_aarch64_##N##X
499 #define CF1(N, X) CODE_FOR_##N##X##1
500 #define CF2(N, X) CODE_FOR_##N##X##2
501 #define CF3(N, X) CODE_FOR_##N##X##3
502 #define CF4(N, X) CODE_FOR_##N##X##4
503 #define CF10(N, X) CODE_FOR_##N##X
505 /* Define cascading VAR<N> macros that are used from
506 aarch64-builtin-iterators.h to iterate over modes. These definitions
507 will end up generating a number of VAR1 expansions and code later on in the
508 file should redefine VAR1 to whatever it needs to process on a per-mode
510 #define VAR2(T, N, MAP, FLAG, A, B) \
511 VAR1 (T, N, MAP, FLAG, A) \
512 VAR1 (T, N, MAP, FLAG, B)
513 #define VAR3(T, N, MAP, FLAG, A, B, C) \
514 VAR2 (T, N, MAP, FLAG, A, B) \
515 VAR1 (T, N, MAP, FLAG, C)
516 #define VAR4(T, N, MAP, FLAG, A, B, C, D) \
517 VAR3 (T, N, MAP, FLAG, A, B, C) \
518 VAR1 (T, N, MAP, FLAG, D)
519 #define VAR5(T, N, MAP, FLAG, A, B, C, D, E) \
520 VAR4 (T, N, MAP, FLAG, A, B, C, D) \
521 VAR1 (T, N, MAP, FLAG, E)
522 #define VAR6(T, N, MAP, FLAG, A, B, C, D, E, F) \
523 VAR5 (T, N, MAP, FLAG, A, B, C, D, E) \
524 VAR1 (T, N, MAP, FLAG, F)
525 #define VAR7(T, N, MAP, FLAG, A, B, C, D, E, F, G) \
526 VAR6 (T, N, MAP, FLAG, A, B, C, D, E, F) \
527 VAR1 (T, N, MAP, FLAG, G)
528 #define VAR8(T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \
529 VAR7 (T, N, MAP, FLAG, A, B, C, D, E, F, G) \
530 VAR1 (T, N, MAP, FLAG, H)
531 #define VAR9(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \
532 VAR8 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \
533 VAR1 (T, N, MAP, FLAG, I)
534 #define VAR10(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \
535 VAR9 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \
536 VAR1 (T, N, MAP, FLAG, J)
537 #define VAR11(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \
538 VAR10 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \
539 VAR1 (T, N, MAP, FLAG, K)
540 #define VAR12(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \
541 VAR11 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \
542 VAR1 (T, N, MAP, FLAG, L)
543 #define VAR13(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \
544 VAR12 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \
545 VAR1 (T, N, MAP, FLAG, M)
546 #define VAR14(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
547 VAR13 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \
548 VAR1 (T, X, MAP, FLAG, N)
549 #define VAR15(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
550 VAR14 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
551 VAR1 (T, X, MAP, FLAG, O)
552 #define VAR16(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) \
553 VAR15 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
554 VAR1 (T, X, MAP, FLAG, P)
556 #include "aarch64-builtin-iterators.h"
558 /* The builtins below should be expanded through the standard optabs
559 CODE_FOR_[u]avg<mode>3_[floor,ceil]. However the mapping scheme in
560 aarch64-simd-builtins.def does not easily allow us to have a pre-mode
561 ("uavg") and post-mode string ("_ceil") in the CODE_FOR_* construction.
562 So the builtins use a name that is natural for AArch64 instructions
563 e.g. "aarch64_srhadd<mode>" and we re-map these to the optab-related
566 #define VAR1(F,T1,T2,I,M) \
567 constexpr insn_code CODE_FOR_aarch64_##F##M = CODE_FOR_##T1##M##3##T2;
569 BUILTIN_VDQ_BHSI (srhadd
, avg
, _ceil
, 0)
570 BUILTIN_VDQ_BHSI (urhadd
, uavg
, _ceil
, 0)
571 BUILTIN_VDQ_BHSI (shadd
, avg
, _floor
, 0)
572 BUILTIN_VDQ_BHSI (uhadd
, uavg
, _floor
, 0)
575 #define VAR1(T, N, MAP, FLAG, A) \
576 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T, FLAG_##FLAG},
578 static aarch64_simd_builtin_datum aarch64_simd_builtin_data
[] = {
579 #include "aarch64-simd-builtins.def"
582 /* There's only 8 CRC32 builtins. Probably not worth their own .def file. */
583 #define AARCH64_CRC32_BUILTINS \
584 CRC32_BUILTIN (crc32b, QI) \
585 CRC32_BUILTIN (crc32h, HI) \
586 CRC32_BUILTIN (crc32w, SI) \
587 CRC32_BUILTIN (crc32x, DI) \
588 CRC32_BUILTIN (crc32cb, QI) \
589 CRC32_BUILTIN (crc32ch, HI) \
590 CRC32_BUILTIN (crc32cw, SI) \
591 CRC32_BUILTIN (crc32cx, DI)
593 /* The next 8 FCMLA instrinsics require some special handling compared the
594 normal simd intrinsics. */
595 #define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \
596 FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \
597 FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \
598 FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \
599 FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \
600 FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \
601 FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \
602 FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \
603 FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \
606 /* vreinterpret intrinsics are defined for any pair of element types.
608 { _f16 _f32 _f64 } { _f16 _f32 _f64 }
609 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 }
610 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }
611 { _p8 _p16 _p64 } { _p8 _p16 _p64 }. */
612 #define VREINTERPRET_BUILTIN2(A, B) \
613 VREINTERPRET_BUILTIN (A, B, d)
615 #define VREINTERPRET_BUILTINS1(A) \
616 VREINTERPRET_BUILTIN2 (A, bf16) \
617 VREINTERPRET_BUILTIN2 (A, f16) \
618 VREINTERPRET_BUILTIN2 (A, f32) \
619 VREINTERPRET_BUILTIN2 (A, f64) \
620 VREINTERPRET_BUILTIN2 (A, s8) \
621 VREINTERPRET_BUILTIN2 (A, s16) \
622 VREINTERPRET_BUILTIN2 (A, s32) \
623 VREINTERPRET_BUILTIN2 (A, s64) \
624 VREINTERPRET_BUILTIN2 (A, u8) \
625 VREINTERPRET_BUILTIN2 (A, u16) \
626 VREINTERPRET_BUILTIN2 (A, u32) \
627 VREINTERPRET_BUILTIN2 (A, u64) \
628 VREINTERPRET_BUILTIN2 (A, p8) \
629 VREINTERPRET_BUILTIN2 (A, p16) \
630 VREINTERPRET_BUILTIN2 (A, p64)
632 #define VREINTERPRET_BUILTINS \
633 VREINTERPRET_BUILTINS1 (bf16) \
634 VREINTERPRET_BUILTINS1 (f16) \
635 VREINTERPRET_BUILTINS1 (f32) \
636 VREINTERPRET_BUILTINS1 (f64) \
637 VREINTERPRET_BUILTINS1 (s8) \
638 VREINTERPRET_BUILTINS1 (s16) \
639 VREINTERPRET_BUILTINS1 (s32) \
640 VREINTERPRET_BUILTINS1 (s64) \
641 VREINTERPRET_BUILTINS1 (u8) \
642 VREINTERPRET_BUILTINS1 (u16) \
643 VREINTERPRET_BUILTINS1 (u32) \
644 VREINTERPRET_BUILTINS1 (u64) \
645 VREINTERPRET_BUILTINS1 (p8) \
646 VREINTERPRET_BUILTINS1 (p16) \
647 VREINTERPRET_BUILTINS1 (p64)
649 /* vreinterpretq intrinsics are additionally defined for p128.
651 { _f16 _f32 _f64 } { _f16 _f32 _f64 }
652 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 }
653 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }
654 { _p8 _p16 _p64 _p128 } { _p8 _p16 _p64 _p128 }. */
655 #define VREINTERPRETQ_BUILTIN2(A, B) \
656 VREINTERPRET_BUILTIN (A, B, q)
658 #define VREINTERPRETQ_BUILTINS1(A) \
659 VREINTERPRETQ_BUILTIN2 (A, bf16) \
660 VREINTERPRETQ_BUILTIN2 (A, f16) \
661 VREINTERPRETQ_BUILTIN2 (A, f32) \
662 VREINTERPRETQ_BUILTIN2 (A, f64) \
663 VREINTERPRETQ_BUILTIN2 (A, s8) \
664 VREINTERPRETQ_BUILTIN2 (A, s16) \
665 VREINTERPRETQ_BUILTIN2 (A, s32) \
666 VREINTERPRETQ_BUILTIN2 (A, s64) \
667 VREINTERPRETQ_BUILTIN2 (A, u8) \
668 VREINTERPRETQ_BUILTIN2 (A, u16) \
669 VREINTERPRETQ_BUILTIN2 (A, u32) \
670 VREINTERPRETQ_BUILTIN2 (A, u64) \
671 VREINTERPRETQ_BUILTIN2 (A, p8) \
672 VREINTERPRETQ_BUILTIN2 (A, p16) \
673 VREINTERPRETQ_BUILTIN2 (A, p64) \
674 VREINTERPRETQ_BUILTIN2 (A, p128)
676 #define VREINTERPRETQ_BUILTINS \
677 VREINTERPRETQ_BUILTINS1 (bf16) \
678 VREINTERPRETQ_BUILTINS1 (f16) \
679 VREINTERPRETQ_BUILTINS1 (f32) \
680 VREINTERPRETQ_BUILTINS1 (f64) \
681 VREINTERPRETQ_BUILTINS1 (s8) \
682 VREINTERPRETQ_BUILTINS1 (s16) \
683 VREINTERPRETQ_BUILTINS1 (s32) \
684 VREINTERPRETQ_BUILTINS1 (s64) \
685 VREINTERPRETQ_BUILTINS1 (u8) \
686 VREINTERPRETQ_BUILTINS1 (u16) \
687 VREINTERPRETQ_BUILTINS1 (u32) \
688 VREINTERPRETQ_BUILTINS1 (u64) \
689 VREINTERPRETQ_BUILTINS1 (p8) \
690 VREINTERPRETQ_BUILTINS1 (p16) \
691 VREINTERPRETQ_BUILTINS1 (p64) \
692 VREINTERPRETQ_BUILTINS1 (p128)
694 #define AARCH64_SIMD_VREINTERPRET_BUILTINS \
695 VREINTERPRET_BUILTINS \
696 VREINTERPRETQ_BUILTINS
702 const enum insn_code icode
;
704 } aarch64_crc_builtin_datum
;
706 /* Hold information about how to expand the FCMLA_LANEQ builtins. */
711 const enum insn_code icode
;
714 } aarch64_fcmla_laneq_builtin_datum
;
716 /* Hold information about how to declare SIMD intrinsics. */
721 unsigned int op_count
;
722 machine_mode op_modes
[SIMD_MAX_BUILTIN_ARGS
];
723 enum aarch64_type_qualifiers qualifiers
[SIMD_MAX_BUILTIN_ARGS
];
726 } aarch64_simd_intrinsic_datum
;
728 #define CRC32_BUILTIN(N, M) \
731 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
732 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M,
734 #define VREINTERPRET_BUILTIN(A, B, L) \
735 AARCH64_SIMD_BUILTIN_VREINTERPRET##L##_##A##_##B,
738 #define VAR1(T, N, MAP, FLAG, A) \
739 AARCH64_SIMD_BUILTIN_##T##_##N##A,
741 enum aarch64_builtins
745 AARCH64_BUILTIN_GET_FPCR
,
746 AARCH64_BUILTIN_SET_FPCR
,
747 AARCH64_BUILTIN_GET_FPSR
,
748 AARCH64_BUILTIN_SET_FPSR
,
750 AARCH64_BUILTIN_GET_FPCR64
,
751 AARCH64_BUILTIN_SET_FPCR64
,
752 AARCH64_BUILTIN_GET_FPSR64
,
753 AARCH64_BUILTIN_SET_FPSR64
,
755 AARCH64_BUILTIN_RSQRT_DF
,
756 AARCH64_BUILTIN_RSQRT_SF
,
757 AARCH64_BUILTIN_RSQRT_V2DF
,
758 AARCH64_BUILTIN_RSQRT_V2SF
,
759 AARCH64_BUILTIN_RSQRT_V4SF
,
760 AARCH64_SIMD_BUILTIN_BASE
,
761 AARCH64_SIMD_BUILTIN_LANE_CHECK
,
762 #include "aarch64-simd-builtins.def"
763 /* The first enum element which is based on an insn_data pattern. */
764 AARCH64_SIMD_PATTERN_START
= AARCH64_SIMD_BUILTIN_LANE_CHECK
+ 1,
765 AARCH64_SIMD_BUILTIN_MAX
= AARCH64_SIMD_PATTERN_START
766 + ARRAY_SIZE (aarch64_simd_builtin_data
) - 1,
767 AARCH64_CRC32_BUILTIN_BASE
,
768 AARCH64_CRC32_BUILTINS
769 AARCH64_CRC32_BUILTIN_MAX
,
770 /* SIMD intrinsic builtins. */
771 AARCH64_SIMD_VREINTERPRET_BUILTINS
772 /* ARMv8.3-A Pointer Authentication Builtins. */
773 AARCH64_PAUTH_BUILTIN_AUTIA1716
,
774 AARCH64_PAUTH_BUILTIN_PACIA1716
,
775 AARCH64_PAUTH_BUILTIN_AUTIB1716
,
776 AARCH64_PAUTH_BUILTIN_PACIB1716
,
777 AARCH64_PAUTH_BUILTIN_XPACLRI
,
778 /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins. */
779 AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE
,
780 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
781 /* Builtin for Arm8.3-a Javascript conversion instruction. */
784 AARCH64_TME_BUILTIN_TSTART
,
785 AARCH64_TME_BUILTIN_TCOMMIT
,
786 AARCH64_TME_BUILTIN_TTEST
,
787 AARCH64_TME_BUILTIN_TCANCEL
,
788 /* Armv8.5-a RNG instruction builtins. */
789 AARCH64_BUILTIN_RNG_RNDR
,
790 AARCH64_BUILTIN_RNG_RNDRRS
,
791 /* MEMTAG builtins. */
792 AARCH64_MEMTAG_BUILTIN_START
,
793 AARCH64_MEMTAG_BUILTIN_IRG
,
794 AARCH64_MEMTAG_BUILTIN_GMI
,
795 AARCH64_MEMTAG_BUILTIN_SUBP
,
796 AARCH64_MEMTAG_BUILTIN_INC_TAG
,
797 AARCH64_MEMTAG_BUILTIN_SET_TAG
,
798 AARCH64_MEMTAG_BUILTIN_GET_TAG
,
799 AARCH64_MEMTAG_BUILTIN_END
,
801 AARCH64_LS64_BUILTIN_LD64B
,
802 AARCH64_LS64_BUILTIN_ST64B
,
803 AARCH64_LS64_BUILTIN_ST64BV
,
804 AARCH64_LS64_BUILTIN_ST64BV0
,
815 #define CRC32_BUILTIN(N, M) \
816 {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
818 static aarch64_crc_builtin_datum aarch64_crc_builtin_data
[] = {
819 AARCH64_CRC32_BUILTINS
823 #undef FCMLA_LANEQ_BUILTIN
824 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
825 {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \
826 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T},
828 /* This structure contains how to manage the mapping form the builtin to the
829 instruction to generate in the backend and how to invoke the instruction. */
830 static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data
[] = {
831 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
834 #undef VREINTERPRET_BUILTIN
835 #define VREINTERPRET_BUILTIN(A, B, L) \
836 {"vreinterpret" SIMD_INTR_LENGTH_CHAR(L) "_" #A "_" #B, \
837 AARCH64_SIMD_BUILTIN_VREINTERPRET##L##_##A##_##B, \
839 { SIMD_INTR_MODE(A, L), SIMD_INTR_MODE(B, L) }, \
840 { SIMD_INTR_QUAL(A), SIMD_INTR_QUAL(B) }, \
842 SIMD_INTR_MODE(A, L) == SIMD_INTR_MODE(B, L) \
843 && SIMD_INTR_QUAL(A) == SIMD_INTR_QUAL(B) \
846 static const aarch64_simd_intrinsic_datum aarch64_simd_intrinsic_data
[] = {
847 AARCH64_SIMD_VREINTERPRET_BUILTINS
853 static GTY(()) tree aarch64_builtin_decls
[AARCH64_BUILTIN_MAX
];
855 #define NUM_DREG_TYPES 6
856 #define NUM_QREG_TYPES 6
858 /* Internal scalar builtin types. These types are used to support
859 neon intrinsic builtins. They are _not_ user-visible types. Therefore
860 the mangling for these types are implementation defined. */
861 const char *aarch64_scalar_builtin_types
[] = {
862 "__builtin_aarch64_simd_qi",
863 "__builtin_aarch64_simd_hi",
864 "__builtin_aarch64_simd_si",
865 "__builtin_aarch64_simd_hf",
866 "__builtin_aarch64_simd_sf",
867 "__builtin_aarch64_simd_di",
868 "__builtin_aarch64_simd_df",
869 "__builtin_aarch64_simd_poly8",
870 "__builtin_aarch64_simd_poly16",
871 "__builtin_aarch64_simd_poly64",
872 "__builtin_aarch64_simd_poly128",
873 "__builtin_aarch64_simd_ti",
874 "__builtin_aarch64_simd_uqi",
875 "__builtin_aarch64_simd_uhi",
876 "__builtin_aarch64_simd_usi",
877 "__builtin_aarch64_simd_udi",
878 "__builtin_aarch64_simd_ei",
879 "__builtin_aarch64_simd_oi",
880 "__builtin_aarch64_simd_ci",
881 "__builtin_aarch64_simd_xi",
882 "__builtin_aarch64_simd_bf",
886 #define ENTRY(E, M, Q, G) E,
887 enum aarch64_simd_type
889 #include "aarch64-simd-builtin-types.def"
890 ARM_NEON_H_TYPES_LAST
894 struct GTY(()) aarch64_simd_type_info
896 enum aarch64_simd_type type
;
898 /* Internal type name. */
901 /* Internal type name(mangled). The mangled names conform to the
902 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
903 Appendix A). To qualify for emission with the mangled names defined in
904 that document, a vector type must not only be of the correct mode but also
905 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
906 types are registered by aarch64_init_simd_builtin_types (). In other
907 words, vector types defined in other ways e.g. via vector_size attribute
908 will get default mangled names. */
917 /* Machine mode the internal type maps to. */
918 enum machine_mode mode
;
921 enum aarch64_type_qualifiers q
;
924 #define ENTRY(E, M, Q, G) \
925 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
926 static GTY(()) struct aarch64_simd_type_info aarch64_simd_types
[] = {
927 #include "aarch64-simd-builtin-types.def"
931 static machine_mode aarch64_simd_tuple_modes
[ARM_NEON_H_TYPES_LAST
][3];
932 static GTY(()) tree aarch64_simd_tuple_types
[ARM_NEON_H_TYPES_LAST
][3];
934 static GTY(()) tree aarch64_simd_intOI_type_node
= NULL_TREE
;
935 static GTY(()) tree aarch64_simd_intCI_type_node
= NULL_TREE
;
936 static GTY(()) tree aarch64_simd_intXI_type_node
= NULL_TREE
;
938 /* The user-visible __fp16 type, and a pointer to that type. Used
939 across the back-end. */
940 tree aarch64_fp16_type_node
= NULL_TREE
;
941 tree aarch64_fp16_ptr_type_node
= NULL_TREE
;
943 /* Back-end node type for brain float (bfloat) types. */
944 tree aarch64_bf16_ptr_type_node
= NULL_TREE
;
946 /* Wrapper around add_builtin_function. NAME is the name of the built-in
947 function, TYPE is the function type, CODE is the function subcode
948 (relative to AARCH64_BUILTIN_GENERAL), and ATTRS is the function
951 aarch64_general_add_builtin (const char *name
, tree type
, unsigned int code
,
952 tree attrs
= NULL_TREE
)
954 code
= (code
<< AARCH64_BUILTIN_SHIFT
) | AARCH64_BUILTIN_GENERAL
;
955 return add_builtin_function (name
, type
, code
, BUILT_IN_MD
,
960 aarch64_mangle_builtin_scalar_type (const_tree type
)
964 while (aarch64_scalar_builtin_types
[i
] != NULL
)
966 const char *name
= aarch64_scalar_builtin_types
[i
];
968 if (TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
969 && DECL_NAME (TYPE_NAME (type
))
970 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type
))), name
))
971 return aarch64_scalar_builtin_types
[i
];
978 aarch64_mangle_builtin_vector_type (const_tree type
)
980 tree attrs
= TYPE_ATTRIBUTES (type
);
981 if (tree attr
= lookup_attribute ("Advanced SIMD type", attrs
))
983 tree mangled_name
= TREE_VALUE (TREE_VALUE (attr
));
984 return IDENTIFIER_POINTER (mangled_name
);
991 aarch64_general_mangle_builtin_type (const_tree type
)
994 /* Walk through all the AArch64 builtins types tables to filter out the
996 if ((mangle
= aarch64_mangle_builtin_vector_type (type
))
997 || (mangle
= aarch64_mangle_builtin_scalar_type (type
)))
1003 /* Helper function for aarch64_simd_builtin_type. */
1005 aarch64_int_or_fp_type (machine_mode mode
,
1006 enum aarch64_type_qualifiers qualifiers
)
1008 #define QUAL_TYPE(M) ((qualifiers & qualifier_unsigned) \
1009 ? unsigned_int##M##_type_node : int##M##_type_node);
1013 return QUAL_TYPE (QI
);
1015 return QUAL_TYPE (HI
);
1017 return QUAL_TYPE (SI
);
1019 return QUAL_TYPE (DI
);
1021 return QUAL_TYPE (TI
);
1023 return aarch64_simd_intOI_type_node
;
1025 return aarch64_simd_intCI_type_node
;
1027 return aarch64_simd_intXI_type_node
;
1029 return aarch64_fp16_type_node
;
1031 return float_type_node
;
1033 return double_type_node
;
1035 return bfloat16_type_node
;
1042 /* Helper function for aarch64_simd_builtin_type. */
1044 aarch64_lookup_simd_type_in_table (machine_mode mode
,
1045 enum aarch64_type_qualifiers qualifiers
)
1048 int nelts
= ARRAY_SIZE (aarch64_simd_types
);
1049 int q
= qualifiers
& (qualifier_poly
| qualifier_unsigned
);
1051 for (i
= 0; i
< nelts
; i
++)
1053 if (aarch64_simd_types
[i
].mode
== mode
1054 && aarch64_simd_types
[i
].q
== q
)
1055 return aarch64_simd_types
[i
].itype
;
1056 if (aarch64_simd_tuple_types
[i
][0] != NULL_TREE
)
1057 for (int j
= 0; j
< 3; j
++)
1058 if (aarch64_simd_tuple_modes
[i
][j
] == mode
1059 && aarch64_simd_types
[i
].q
== q
)
1060 return aarch64_simd_tuple_types
[i
][j
];
1066 /* Return a type for an operand with specified mode and qualifiers. */
1068 aarch64_simd_builtin_type (machine_mode mode
,
1069 enum aarch64_type_qualifiers qualifiers
)
1071 tree type
= NULL_TREE
;
1073 /* For pointers, we want a pointer to the basic type of the vector. */
1074 if ((qualifiers
& qualifier_pointer
) && VECTOR_MODE_P (mode
))
1075 mode
= GET_MODE_INNER (mode
);
1077 /* Non-poly scalar modes map to standard types not in the table. */
1078 if ((qualifiers
& qualifier_poly
) || VECTOR_MODE_P (mode
))
1079 type
= aarch64_lookup_simd_type_in_table (mode
, qualifiers
);
1081 type
= aarch64_int_or_fp_type (mode
, qualifiers
);
1083 gcc_assert (type
!= NULL_TREE
);
1085 /* Add qualifiers. */
1086 if (qualifiers
& qualifier_const
)
1087 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1088 if (qualifiers
& qualifier_pointer
)
1089 type
= build_pointer_type (type
);
1095 aarch64_init_simd_builtin_types (void)
1098 int nelts
= ARRAY_SIZE (aarch64_simd_types
);
1101 /* Init all the element types built by the front-end. */
1102 aarch64_simd_types
[Int8x8_t
].eltype
= intQI_type_node
;
1103 aarch64_simd_types
[Int8x16_t
].eltype
= intQI_type_node
;
1104 aarch64_simd_types
[Int16x4_t
].eltype
= intHI_type_node
;
1105 aarch64_simd_types
[Int16x8_t
].eltype
= intHI_type_node
;
1106 aarch64_simd_types
[Int32x2_t
].eltype
= intSI_type_node
;
1107 aarch64_simd_types
[Int32x4_t
].eltype
= intSI_type_node
;
1108 aarch64_simd_types
[Int64x1_t
].eltype
= intDI_type_node
;
1109 aarch64_simd_types
[Int64x2_t
].eltype
= intDI_type_node
;
1110 aarch64_simd_types
[Uint8x8_t
].eltype
= unsigned_intQI_type_node
;
1111 aarch64_simd_types
[Uint8x16_t
].eltype
= unsigned_intQI_type_node
;
1112 aarch64_simd_types
[Uint16x4_t
].eltype
= unsigned_intHI_type_node
;
1113 aarch64_simd_types
[Uint16x8_t
].eltype
= unsigned_intHI_type_node
;
1114 aarch64_simd_types
[Uint32x2_t
].eltype
= unsigned_intSI_type_node
;
1115 aarch64_simd_types
[Uint32x4_t
].eltype
= unsigned_intSI_type_node
;
1116 aarch64_simd_types
[Uint64x1_t
].eltype
= unsigned_intDI_type_node
;
1117 aarch64_simd_types
[Uint64x2_t
].eltype
= unsigned_intDI_type_node
;
1119 /* Poly types are a world of their own. */
1120 aarch64_simd_types
[Poly8_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
=
1121 build_distinct_type_copy (unsigned_intQI_type_node
);
1122 /* Prevent front-ends from transforming Poly8_t arrays into string
1124 TYPE_STRING_FLAG (aarch64_simd_types
[Poly8_t
].eltype
) = false;
1126 aarch64_simd_types
[Poly16_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
=
1127 build_distinct_type_copy (unsigned_intHI_type_node
);
1128 aarch64_simd_types
[Poly64_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
=
1129 build_distinct_type_copy (unsigned_intDI_type_node
);
1130 aarch64_simd_types
[Poly128_t
].eltype
= aarch64_simd_types
[Poly128_t
].itype
=
1131 build_distinct_type_copy (unsigned_intTI_type_node
);
1132 /* Init poly vector element types with scalar poly types. */
1133 aarch64_simd_types
[Poly8x8_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
;
1134 aarch64_simd_types
[Poly8x16_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
;
1135 aarch64_simd_types
[Poly16x4_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
;
1136 aarch64_simd_types
[Poly16x8_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
;
1137 aarch64_simd_types
[Poly64x1_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
;
1138 aarch64_simd_types
[Poly64x2_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
;
1140 /* Continue with standard types. */
1141 aarch64_simd_types
[Float16x4_t
].eltype
= aarch64_fp16_type_node
;
1142 aarch64_simd_types
[Float16x8_t
].eltype
= aarch64_fp16_type_node
;
1143 aarch64_simd_types
[Float32x2_t
].eltype
= float_type_node
;
1144 aarch64_simd_types
[Float32x4_t
].eltype
= float_type_node
;
1145 aarch64_simd_types
[Float64x1_t
].eltype
= double_type_node
;
1146 aarch64_simd_types
[Float64x2_t
].eltype
= double_type_node
;
1148 /* Init Bfloat vector types with underlying __bf16 type. */
1149 aarch64_simd_types
[Bfloat16x4_t
].eltype
= bfloat16_type_node
;
1150 aarch64_simd_types
[Bfloat16x8_t
].eltype
= bfloat16_type_node
;
1152 for (i
= 0; i
< nelts
; i
++)
1154 tree eltype
= aarch64_simd_types
[i
].eltype
;
1155 machine_mode mode
= aarch64_simd_types
[i
].mode
;
1157 if (aarch64_simd_types
[i
].itype
== NULL
)
1159 tree type
= build_vector_type (eltype
, GET_MODE_NUNITS (mode
));
1160 type
= build_distinct_type_copy (type
);
1161 SET_TYPE_STRUCTURAL_EQUALITY (type
);
1163 tree mangled_name
= get_identifier (aarch64_simd_types
[i
].mangle
);
1164 tree value
= tree_cons (NULL_TREE
, mangled_name
, NULL_TREE
);
1165 TYPE_ATTRIBUTES (type
)
1166 = tree_cons (get_identifier ("Advanced SIMD type"), value
,
1167 TYPE_ATTRIBUTES (type
));
1168 aarch64_simd_types
[i
].itype
= type
;
1171 tdecl
= add_builtin_type (aarch64_simd_types
[i
].name
,
1172 aarch64_simd_types
[i
].itype
);
1173 TYPE_NAME (aarch64_simd_types
[i
].itype
) = tdecl
;
1176 #define AARCH64_BUILD_SIGNED_TYPE(mode) \
1177 make_signed_type (GET_MODE_PRECISION (mode));
1178 aarch64_simd_intOI_type_node
= AARCH64_BUILD_SIGNED_TYPE (OImode
);
1179 aarch64_simd_intCI_type_node
= AARCH64_BUILD_SIGNED_TYPE (CImode
);
1180 aarch64_simd_intXI_type_node
= AARCH64_BUILD_SIGNED_TYPE (XImode
);
1181 #undef AARCH64_BUILD_SIGNED_TYPE
1183 tdecl
= add_builtin_type
1184 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node
);
1185 TYPE_NAME (aarch64_simd_intOI_type_node
) = tdecl
;
1186 tdecl
= add_builtin_type
1187 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node
);
1188 TYPE_NAME (aarch64_simd_intCI_type_node
) = tdecl
;
1189 tdecl
= add_builtin_type
1190 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node
);
1191 TYPE_NAME (aarch64_simd_intXI_type_node
) = tdecl
;
1195 aarch64_init_simd_builtin_scalar_types (void)
1197 /* Define typedefs for all the standard scalar types. */
1198 (*lang_hooks
.types
.register_builtin_type
) (intQI_type_node
,
1199 "__builtin_aarch64_simd_qi");
1200 (*lang_hooks
.types
.register_builtin_type
) (intHI_type_node
,
1201 "__builtin_aarch64_simd_hi");
1202 (*lang_hooks
.types
.register_builtin_type
) (aarch64_fp16_type_node
,
1203 "__builtin_aarch64_simd_hf");
1204 (*lang_hooks
.types
.register_builtin_type
) (intSI_type_node
,
1205 "__builtin_aarch64_simd_si");
1206 (*lang_hooks
.types
.register_builtin_type
) (float_type_node
,
1207 "__builtin_aarch64_simd_sf");
1208 (*lang_hooks
.types
.register_builtin_type
) (intDI_type_node
,
1209 "__builtin_aarch64_simd_di");
1210 (*lang_hooks
.types
.register_builtin_type
) (double_type_node
,
1211 "__builtin_aarch64_simd_df");
1212 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intQI_type_node
,
1213 "__builtin_aarch64_simd_poly8");
1214 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intHI_type_node
,
1215 "__builtin_aarch64_simd_poly16");
1216 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intDI_type_node
,
1217 "__builtin_aarch64_simd_poly64");
1218 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intTI_type_node
,
1219 "__builtin_aarch64_simd_poly128");
1220 (*lang_hooks
.types
.register_builtin_type
) (intTI_type_node
,
1221 "__builtin_aarch64_simd_ti");
1222 (*lang_hooks
.types
.register_builtin_type
) (bfloat16_type_node
,
1223 "__builtin_aarch64_simd_bf");
1224 /* Unsigned integer types for various mode sizes. */
1225 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intQI_type_node
,
1226 "__builtin_aarch64_simd_uqi");
1227 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intHI_type_node
,
1228 "__builtin_aarch64_simd_uhi");
1229 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intSI_type_node
,
1230 "__builtin_aarch64_simd_usi");
1231 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intDI_type_node
,
1232 "__builtin_aarch64_simd_udi");
1235 /* Return a set of FLAG_* flags derived from FLAGS
1236 that describe what a function with result MODE could do,
1237 taking the command-line flags into account. */
1239 aarch64_call_properties (unsigned int flags
, machine_mode mode
)
1241 if (!(flags
& FLAG_AUTO_FP
) && FLOAT_MODE_P (mode
))
1244 /* -fno-trapping-math means that we can assume any FP exceptions
1245 are not user-visible. */
1246 if (!flag_trapping_math
)
1247 flags
&= ~FLAG_RAISE_FP_EXCEPTIONS
;
1252 /* Return true if calls to a function with flags F and mode MODE
1253 could modify some form of global state. */
1255 aarch64_modifies_global_state_p (unsigned int f
, machine_mode mode
)
1257 unsigned int flags
= aarch64_call_properties (f
, mode
);
1259 if (flags
& FLAG_RAISE_FP_EXCEPTIONS
)
1262 if (flags
& FLAG_PREFETCH_MEMORY
)
1265 return flags
& FLAG_WRITE_MEMORY
;
1268 /* Return true if calls to a function with flags F and mode MODE
1269 could read some form of global state. */
1271 aarch64_reads_global_state_p (unsigned int f
, machine_mode mode
)
1273 unsigned int flags
= aarch64_call_properties (f
, mode
);
1275 if (flags
& FLAG_READ_FPCR
)
1278 return flags
& FLAG_READ_MEMORY
;
1281 /* Return true if calls to a function with flags F and mode MODE
1282 could raise a signal. */
1284 aarch64_could_trap_p (unsigned int f
, machine_mode mode
)
1286 unsigned int flags
= aarch64_call_properties (f
, mode
);
1288 if (flags
& FLAG_RAISE_FP_EXCEPTIONS
)
1291 if (flags
& (FLAG_READ_MEMORY
| FLAG_WRITE_MEMORY
))
1297 /* Add attribute NAME to ATTRS. */
1299 aarch64_add_attribute (const char *name
, tree attrs
)
1301 return tree_cons (get_identifier (name
), NULL_TREE
, attrs
);
1304 /* Return the appropriate attributes for a function that has
1305 flags F and mode MODE. */
1307 aarch64_get_attributes (unsigned int f
, machine_mode mode
)
1309 tree attrs
= NULL_TREE
;
1311 if (!aarch64_modifies_global_state_p (f
, mode
))
1313 if (aarch64_reads_global_state_p (f
, mode
))
1314 attrs
= aarch64_add_attribute ("pure", attrs
);
1316 attrs
= aarch64_add_attribute ("const", attrs
);
1319 if (!flag_non_call_exceptions
|| !aarch64_could_trap_p (f
, mode
))
1320 attrs
= aarch64_add_attribute ("nothrow", attrs
);
1322 return aarch64_add_attribute ("leaf", attrs
);
1325 /* Due to the architecture not providing lane variant of the lane instructions
1326 for fcmla we can't use the standard simd builtin expansion code, but we
1327 still want the majority of the validation that would normally be done. */
1330 aarch64_init_fcmla_laneq_builtins (void)
1334 for (i
= 0; i
< ARRAY_SIZE (aarch64_fcmla_lane_builtin_data
); ++i
)
1336 aarch64_fcmla_laneq_builtin_datum
* d
1337 = &aarch64_fcmla_lane_builtin_data
[i
];
1338 tree argtype
= aarch64_simd_builtin_type (d
->mode
, qualifier_none
);
1339 machine_mode quadmode
= GET_MODE_2XWIDER_MODE (d
->mode
).require ();
1340 tree quadtype
= aarch64_simd_builtin_type (quadmode
, qualifier_none
);
1342 = aarch64_simd_builtin_type (SImode
, qualifier_lane_pair_index
);
1343 tree ftype
= build_function_type_list (argtype
, argtype
, argtype
,
1344 quadtype
, lanetype
, NULL_TREE
);
1345 tree attrs
= aarch64_get_attributes (FLAG_FP
, d
->mode
);
1347 = aarch64_general_add_builtin (d
->name
, ftype
, d
->fcode
, attrs
);
1349 aarch64_builtin_decls
[d
->fcode
] = fndecl
;
1354 aarch64_init_simd_intrinsics (void)
1358 for (i
= 0; i
< ARRAY_SIZE (aarch64_simd_intrinsic_data
); ++i
)
1360 auto d
= &aarch64_simd_intrinsic_data
[i
];
1365 tree return_type
= void_type_node
;
1366 tree args
= void_list_node
;
1368 for (int op_num
= d
->op_count
- 1; op_num
>= 0; op_num
--)
1370 machine_mode op_mode
= d
->op_modes
[op_num
];
1371 enum aarch64_type_qualifiers qualifiers
= d
->qualifiers
[op_num
];
1373 tree eltype
= aarch64_simd_builtin_type (op_mode
, qualifiers
);
1376 return_type
= eltype
;
1378 args
= tree_cons (NULL_TREE
, eltype
, args
);
1381 tree ftype
= build_function_type (return_type
, args
);
1382 tree attrs
= aarch64_get_attributes (d
->flags
, d
->op_modes
[0]);
1384 = (d
->fcode
<< AARCH64_BUILTIN_SHIFT
| AARCH64_BUILTIN_GENERAL
);
1385 tree fndecl
= simulate_builtin_function_decl (input_location
, d
->name
,
1386 ftype
, code
, NULL
, attrs
);
1387 aarch64_builtin_decls
[d
->fcode
] = fndecl
;
1392 aarch64_init_simd_builtin_functions (bool called_from_pragma
)
1394 unsigned int i
, fcode
= AARCH64_SIMD_PATTERN_START
;
1396 if (!called_from_pragma
)
1398 tree lane_check_fpr
= build_function_type_list (void_type_node
,
1403 aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_LANE_CHECK
]
1404 = aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi",
1406 AARCH64_SIMD_BUILTIN_LANE_CHECK
);
1409 for (i
= 0; i
< ARRAY_SIZE (aarch64_simd_builtin_data
); i
++, fcode
++)
1411 bool print_type_signature_p
= false;
1412 char type_signature
[SIMD_MAX_BUILTIN_ARGS
+ 1] = { 0 };
1413 aarch64_simd_builtin_datum
*d
= &aarch64_simd_builtin_data
[i
];
1420 /* We must track two variables here. op_num is
1421 the operand number as in the RTL pattern. This is
1422 required to access the mode (e.g. V4SF mode) of the
1423 argument, from which the base type can be derived.
1424 arg_num is an index in to the qualifiers data, which
1425 gives qualifiers to the type (e.g. const unsigned).
1426 The reason these two variables may differ by one is the
1427 void return type. While all return types take the 0th entry
1428 in the qualifiers array, there is no operand for them in the
1430 int op_num
= insn_data
[d
->code
].n_operands
- 1;
1431 int arg_num
= d
->qualifiers
[0] & qualifier_void
1434 tree return_type
= void_type_node
, args
= void_list_node
;
1437 int struct_mode_args
= 0;
1438 for (int j
= op_num
; j
>= 0; j
--)
1440 machine_mode op_mode
= insn_data
[d
->code
].operand
[j
].mode
;
1441 if (aarch64_advsimd_struct_mode_p (op_mode
))
1445 if ((called_from_pragma
&& struct_mode_args
== 0)
1446 || (!called_from_pragma
&& struct_mode_args
> 0))
1449 /* Build a function type directly from the insn_data for this
1450 builtin. The build_function_type () function takes care of
1451 removing duplicates for us. */
1452 for (; op_num
>= 0; arg_num
--, op_num
--)
1454 machine_mode op_mode
= insn_data
[d
->code
].operand
[op_num
].mode
;
1455 enum aarch64_type_qualifiers qualifiers
= d
->qualifiers
[arg_num
];
1457 if (qualifiers
& qualifier_unsigned
)
1459 type_signature
[op_num
] = 'u';
1460 print_type_signature_p
= true;
1462 else if (qualifiers
& qualifier_poly
)
1464 type_signature
[op_num
] = 'p';
1465 print_type_signature_p
= true;
1468 type_signature
[op_num
] = 's';
1470 /* Some builtins have different user-facing types
1471 for certain arguments, encoded in d->mode. */
1472 if (qualifiers
& qualifier_map_mode
)
1475 eltype
= aarch64_simd_builtin_type (op_mode
, qualifiers
);
1477 /* If we have reached arg_num == 0, we are at a non-void
1478 return type. Otherwise, we are still processing
1481 return_type
= eltype
;
1483 args
= tree_cons (NULL_TREE
, eltype
, args
);
1486 ftype
= build_function_type (return_type
, args
);
1488 gcc_assert (ftype
!= NULL
);
1490 if (print_type_signature_p
)
1491 snprintf (namebuf
, sizeof (namebuf
), "__builtin_aarch64_%s_%s",
1492 d
->name
, type_signature
);
1494 snprintf (namebuf
, sizeof (namebuf
), "__builtin_aarch64_%s",
1497 tree attrs
= aarch64_get_attributes (d
->flags
, d
->mode
);
1499 if (called_from_pragma
)
1501 unsigned int raw_code
1502 = (fcode
<< AARCH64_BUILTIN_SHIFT
) | AARCH64_BUILTIN_GENERAL
;
1503 fndecl
= simulate_builtin_function_decl (input_location
, namebuf
,
1504 ftype
, raw_code
, NULL
,
1508 fndecl
= aarch64_general_add_builtin (namebuf
, ftype
, fcode
, attrs
);
1510 aarch64_builtin_decls
[fcode
] = fndecl
;
1514 /* Register the tuple type that contains NUM_VECTORS of the AdvSIMD type
1515 indexed by TYPE_INDEX. */
1517 register_tuple_type (unsigned int num_vectors
, unsigned int type_index
)
1519 aarch64_simd_type_info
*type
= &aarch64_simd_types
[type_index
];
1521 /* Synthesize the name of the user-visible vector tuple type. */
1522 const char *vector_type_name
= type
->name
;
1523 char tuple_type_name
[sizeof ("bfloat16x4x2_t")];
1524 snprintf (tuple_type_name
, sizeof (tuple_type_name
), "%.*sx%d_t",
1525 (int) strlen (vector_type_name
) - 4, vector_type_name
+ 2,
1527 tuple_type_name
[0] = TOLOWER (tuple_type_name
[0]);
1529 tree vector_type
= type
->itype
;
1530 tree array_type
= build_array_type_nelts (vector_type
, num_vectors
);
1531 if (type
->mode
== DImode
)
1533 if (num_vectors
== 2)
1534 SET_TYPE_MODE (array_type
, V2x1DImode
);
1535 else if (num_vectors
== 3)
1536 SET_TYPE_MODE (array_type
, V3x1DImode
);
1537 else if (num_vectors
== 4)
1538 SET_TYPE_MODE (array_type
, V4x1DImode
);
1541 unsigned int alignment
1542 = known_eq (GET_MODE_SIZE (type
->mode
), 16) ? 128 : 64;
1543 machine_mode tuple_mode
= TYPE_MODE_RAW (array_type
);
1544 gcc_assert (VECTOR_MODE_P (tuple_mode
)
1545 && TYPE_MODE (array_type
) == tuple_mode
1546 && TYPE_ALIGN (array_type
) == alignment
);
1548 tree field
= build_decl (input_location
, FIELD_DECL
,
1549 get_identifier ("val"), array_type
);
1551 tree t
= lang_hooks
.types
.simulate_record_decl (input_location
,
1553 make_array_slice (&field
,
1555 gcc_assert (TYPE_MODE_RAW (t
) == TYPE_MODE (t
)
1556 && (flag_pack_struct
1557 || maximum_field_alignment
1558 || (TYPE_MODE_RAW (t
) == tuple_mode
1559 && TYPE_ALIGN (t
) == alignment
)));
1561 aarch64_simd_tuple_modes
[type_index
][num_vectors
- 2] = tuple_mode
;
1562 aarch64_simd_tuple_types
[type_index
][num_vectors
- 2] = t
;
1566 aarch64_scalar_builtin_type_p (aarch64_simd_type t
)
1568 return (t
== Poly8_t
|| t
== Poly16_t
|| t
== Poly64_t
|| t
== Poly128_t
);
1571 /* Enable AARCH64_FL_* flags EXTRA_FLAGS on top of the base Advanced SIMD
1573 aarch64_simd_switcher::aarch64_simd_switcher (aarch64_feature_flags extra_flags
)
1574 : m_old_asm_isa_flags (aarch64_asm_isa_flags
),
1575 m_old_general_regs_only (TARGET_GENERAL_REGS_ONLY
)
1577 /* Changing the ISA flags should be enough here. We shouldn't need to
1578 pay the compile-time cost of a full target switch. */
1579 global_options
.x_target_flags
&= ~MASK_GENERAL_REGS_ONLY
;
1580 aarch64_set_asm_isa_flags (AARCH64_FL_FP
| AARCH64_FL_SIMD
| extra_flags
);
1583 aarch64_simd_switcher::~aarch64_simd_switcher ()
1585 if (m_old_general_regs_only
)
1586 global_options
.x_target_flags
|= MASK_GENERAL_REGS_ONLY
;
1587 aarch64_set_asm_isa_flags (m_old_asm_isa_flags
);
1590 /* Implement #pragma GCC aarch64 "arm_neon.h".
1592 The types and functions defined here need to be available internally
1593 during LTO as well. */
1595 handle_arm_neon_h (void)
1597 aarch64_simd_switcher simd
;
1599 /* Register the AdvSIMD vector tuple types. */
1600 for (unsigned int i
= 0; i
< ARM_NEON_H_TYPES_LAST
; i
++)
1601 for (unsigned int count
= 2; count
<= 4; ++count
)
1602 if (!aarch64_scalar_builtin_type_p (aarch64_simd_types
[i
].type
))
1603 register_tuple_type (count
, i
);
1605 aarch64_init_simd_builtin_functions (true);
1606 aarch64_init_simd_intrinsics ();
1610 aarch64_init_simd_builtins (void)
1612 aarch64_init_simd_builtin_types ();
1614 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
1615 Therefore we need to preserve the old __builtin scalar types. It can be
1616 removed once all the intrinsics become strongly typed using the qualifier
1618 aarch64_init_simd_builtin_scalar_types ();
1620 aarch64_init_simd_builtin_functions (false);
1622 handle_arm_neon_h ();
1624 /* Initialize the remaining fcmla_laneq intrinsics. */
1625 aarch64_init_fcmla_laneq_builtins ();
1629 aarch64_init_crc32_builtins ()
1631 tree usi_type
= aarch64_simd_builtin_type (SImode
, qualifier_unsigned
);
1634 for (i
= 0; i
< ARRAY_SIZE (aarch64_crc_builtin_data
); ++i
)
1636 aarch64_crc_builtin_datum
* d
= &aarch64_crc_builtin_data
[i
];
1637 tree argtype
= aarch64_simd_builtin_type (d
->mode
, qualifier_unsigned
);
1638 tree ftype
= build_function_type_list (usi_type
, usi_type
, argtype
, NULL_TREE
);
1639 tree attrs
= aarch64_get_attributes (FLAG_NONE
, d
->mode
);
1641 = aarch64_general_add_builtin (d
->name
, ftype
, d
->fcode
, attrs
);
1643 aarch64_builtin_decls
[d
->fcode
] = fndecl
;
1647 /* Add builtins for reciprocal square root. */
1650 aarch64_init_builtin_rsqrt (void)
1655 tree V2SF_type_node
= build_vector_type (float_type_node
, 2);
1656 tree V2DF_type_node
= build_vector_type (double_type_node
, 2);
1657 tree V4SF_type_node
= build_vector_type (float_type_node
, 4);
1659 struct builtin_decls_data
1662 const char *builtin_name
;
1666 builtin_decls_data bdda
[] =
1668 { double_type_node
, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF
},
1669 { float_type_node
, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF
},
1670 { V2DF_type_node
, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF
},
1671 { V2SF_type_node
, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF
},
1672 { V4SF_type_node
, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF
}
1675 builtin_decls_data
*bdd
= bdda
;
1676 builtin_decls_data
*bdd_end
= bdd
+ (ARRAY_SIZE (bdda
));
1678 for (; bdd
< bdd_end
; bdd
++)
1680 ftype
= build_function_type_list (bdd
->type_node
, bdd
->type_node
, NULL_TREE
);
1681 tree attrs
= aarch64_get_attributes (FLAG_FP
, TYPE_MODE (bdd
->type_node
));
1682 fndecl
= aarch64_general_add_builtin (bdd
->builtin_name
,
1683 ftype
, bdd
->function_code
, attrs
);
1684 aarch64_builtin_decls
[bdd
->function_code
] = fndecl
;
1688 /* Initialize the backend types that support the user-visible __fp16
1689 type, also initialize a pointer to that type, to be used when
1693 aarch64_init_fp16_types (void)
1695 aarch64_fp16_type_node
= make_node (REAL_TYPE
);
1696 TYPE_PRECISION (aarch64_fp16_type_node
) = 16;
1697 layout_type (aarch64_fp16_type_node
);
1699 (*lang_hooks
.types
.register_builtin_type
) (aarch64_fp16_type_node
, "__fp16");
1700 aarch64_fp16_ptr_type_node
= build_pointer_type (aarch64_fp16_type_node
);
1703 /* Initialize the backend REAL_TYPE type supporting bfloat types. */
1705 aarch64_init_bf16_types (void)
1707 lang_hooks
.types
.register_builtin_type (bfloat16_type_node
, "__bf16");
1708 aarch64_bf16_ptr_type_node
= build_pointer_type (bfloat16_type_node
);
1711 /* Pointer authentication builtins that will become NOP on legacy platform.
1712 Currently, these builtins are for internal use only (libgcc EH unwinder). */
1715 aarch64_init_pauth_hint_builtins (void)
1717 /* Pointer Authentication builtins. */
1718 tree ftype_pointer_auth
1719 = build_function_type_list (ptr_type_node
, ptr_type_node
,
1720 unsigned_intDI_type_node
, NULL_TREE
);
1721 tree ftype_pointer_strip
1722 = build_function_type_list (ptr_type_node
, ptr_type_node
, NULL_TREE
);
1724 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_AUTIA1716
]
1725 = aarch64_general_add_builtin ("__builtin_aarch64_autia1716",
1727 AARCH64_PAUTH_BUILTIN_AUTIA1716
);
1728 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_PACIA1716
]
1729 = aarch64_general_add_builtin ("__builtin_aarch64_pacia1716",
1731 AARCH64_PAUTH_BUILTIN_PACIA1716
);
1732 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_AUTIB1716
]
1733 = aarch64_general_add_builtin ("__builtin_aarch64_autib1716",
1735 AARCH64_PAUTH_BUILTIN_AUTIB1716
);
1736 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_PACIB1716
]
1737 = aarch64_general_add_builtin ("__builtin_aarch64_pacib1716",
1739 AARCH64_PAUTH_BUILTIN_PACIB1716
);
1740 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_XPACLRI
]
1741 = aarch64_general_add_builtin ("__builtin_aarch64_xpaclri",
1742 ftype_pointer_strip
,
1743 AARCH64_PAUTH_BUILTIN_XPACLRI
);
1746 /* Initialize the transactional memory extension (TME) builtins. */
1748 aarch64_init_tme_builtins (void)
1750 tree ftype_uint64_void
1751 = build_function_type_list (uint64_type_node
, NULL
);
1752 tree ftype_void_void
1753 = build_function_type_list (void_type_node
, NULL
);
1754 tree ftype_void_uint64
1755 = build_function_type_list (void_type_node
, uint64_type_node
, NULL
);
1757 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TSTART
]
1758 = aarch64_general_add_builtin ("__builtin_aarch64_tstart",
1760 AARCH64_TME_BUILTIN_TSTART
);
1761 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TTEST
]
1762 = aarch64_general_add_builtin ("__builtin_aarch64_ttest",
1764 AARCH64_TME_BUILTIN_TTEST
);
1765 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TCOMMIT
]
1766 = aarch64_general_add_builtin ("__builtin_aarch64_tcommit",
1768 AARCH64_TME_BUILTIN_TCOMMIT
);
1769 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TCANCEL
]
1770 = aarch64_general_add_builtin ("__builtin_aarch64_tcancel",
1772 AARCH64_TME_BUILTIN_TCANCEL
);
1775 /* Add builtins for Random Number instructions. */
1778 aarch64_init_rng_builtins (void)
1780 tree unsigned_ptr_type
= build_pointer_type (unsigned_intDI_type_node
);
1782 = build_function_type_list (integer_type_node
, unsigned_ptr_type
, NULL
);
1783 aarch64_builtin_decls
[AARCH64_BUILTIN_RNG_RNDR
]
1784 = aarch64_general_add_builtin ("__builtin_aarch64_rndr", ftype
,
1785 AARCH64_BUILTIN_RNG_RNDR
);
1786 aarch64_builtin_decls
[AARCH64_BUILTIN_RNG_RNDRRS
]
1787 = aarch64_general_add_builtin ("__builtin_aarch64_rndrrs", ftype
,
1788 AARCH64_BUILTIN_RNG_RNDRRS
);
1791 /* Initialize the memory tagging extension (MTE) builtins. */
1795 enum insn_code icode
;
1796 } aarch64_memtag_builtin_data
[AARCH64_MEMTAG_BUILTIN_END
-
1797 AARCH64_MEMTAG_BUILTIN_START
- 1];
1800 aarch64_init_memtag_builtins (void)
1804 #define AARCH64_INIT_MEMTAG_BUILTINS_DECL(F, N, I, T) \
1805 aarch64_builtin_decls[AARCH64_MEMTAG_BUILTIN_##F] \
1806 = aarch64_general_add_builtin ("__builtin_aarch64_memtag_"#N, \
1807 T, AARCH64_MEMTAG_BUILTIN_##F); \
1808 aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_##F - \
1809 AARCH64_MEMTAG_BUILTIN_START - 1] = \
1812 fntype
= build_function_type_list (ptr_type_node
, ptr_type_node
,
1813 uint64_type_node
, NULL
);
1814 AARCH64_INIT_MEMTAG_BUILTINS_DECL (IRG
, irg
, irg
, fntype
);
1816 fntype
= build_function_type_list (uint64_type_node
, ptr_type_node
,
1817 uint64_type_node
, NULL
);
1818 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GMI
, gmi
, gmi
, fntype
);
1820 fntype
= build_function_type_list (ptrdiff_type_node
, ptr_type_node
,
1821 ptr_type_node
, NULL
);
1822 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SUBP
, subp
, subp
, fntype
);
1824 fntype
= build_function_type_list (ptr_type_node
, ptr_type_node
,
1825 unsigned_type_node
, NULL
);
1826 AARCH64_INIT_MEMTAG_BUILTINS_DECL (INC_TAG
, inc_tag
, addg
, fntype
);
1828 fntype
= build_function_type_list (void_type_node
, ptr_type_node
, NULL
);
1829 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SET_TAG
, set_tag
, stg
, fntype
);
1831 fntype
= build_function_type_list (ptr_type_node
, ptr_type_node
, NULL
);
1832 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GET_TAG
, get_tag
, ldg
, fntype
);
1834 #undef AARCH64_INIT_MEMTAG_BUILTINS_DECL
1837 /* Add builtins for Load/store 64 Byte instructions. */
1844 } ls64_builtins_data
;
1846 static GTY(()) tree ls64_arm_data_t
= NULL_TREE
;
1849 aarch64_init_ls64_builtins_types (void)
1855 } __arm_data512_t; */
1856 const char *tuple_type_name
= "__arm_data512_t";
1857 tree node_type
= get_typenode_from_name (UINT64_TYPE
);
1858 tree array_type
= build_array_type_nelts (node_type
, 8);
1859 SET_TYPE_MODE (array_type
, V8DImode
);
1861 gcc_assert (TYPE_MODE_RAW (array_type
) == TYPE_MODE (array_type
));
1862 gcc_assert (TYPE_ALIGN (array_type
) == 64);
1864 tree field
= build_decl (input_location
, FIELD_DECL
,
1865 get_identifier ("val"), array_type
);
1867 ls64_arm_data_t
= lang_hooks
.types
.simulate_record_decl (input_location
,
1869 make_array_slice (&field
, 1));
1871 gcc_assert (TYPE_MODE (ls64_arm_data_t
) == V8DImode
);
1872 gcc_assert (TYPE_MODE_RAW (ls64_arm_data_t
) == TYPE_MODE (ls64_arm_data_t
));
1873 gcc_assert (TYPE_ALIGN (ls64_arm_data_t
) == 64);
1877 aarch64_init_ls64_builtins (void)
1879 aarch64_init_ls64_builtins_types ();
1881 ls64_builtins_data data
[4] = {
1882 {"__builtin_aarch64_ld64b", AARCH64_LS64_BUILTIN_LD64B
,
1883 build_function_type_list (ls64_arm_data_t
,
1884 const_ptr_type_node
, NULL_TREE
)},
1885 {"__builtin_aarch64_st64b", AARCH64_LS64_BUILTIN_ST64B
,
1886 build_function_type_list (void_type_node
, ptr_type_node
,
1887 ls64_arm_data_t
, NULL_TREE
)},
1888 {"__builtin_aarch64_st64bv", AARCH64_LS64_BUILTIN_ST64BV
,
1889 build_function_type_list (uint64_type_node
, ptr_type_node
,
1890 ls64_arm_data_t
, NULL_TREE
)},
1891 {"__builtin_aarch64_st64bv0", AARCH64_LS64_BUILTIN_ST64BV0
,
1892 build_function_type_list (uint64_type_node
, ptr_type_node
,
1893 ls64_arm_data_t
, NULL_TREE
)},
1896 for (size_t i
= 0; i
< ARRAY_SIZE (data
); ++i
)
1897 aarch64_builtin_decls
[data
[i
].code
]
1898 = aarch64_general_add_builtin (data
[i
].name
, data
[i
].type
, data
[i
].code
);
1902 aarch64_init_data_intrinsics (void)
1904 tree uint32_fntype
= build_function_type_list (uint32_type_node
,
1905 uint32_type_node
, NULL_TREE
);
1906 tree ulong_fntype
= build_function_type_list (long_unsigned_type_node
,
1907 long_unsigned_type_node
,
1909 tree uint64_fntype
= build_function_type_list (uint64_type_node
,
1910 uint64_type_node
, NULL_TREE
);
1911 aarch64_builtin_decls
[AARCH64_REV16
]
1912 = aarch64_general_add_builtin ("__builtin_aarch64_rev16", uint32_fntype
,
1914 aarch64_builtin_decls
[AARCH64_REV16L
]
1915 = aarch64_general_add_builtin ("__builtin_aarch64_rev16l", ulong_fntype
,
1917 aarch64_builtin_decls
[AARCH64_REV16LL
]
1918 = aarch64_general_add_builtin ("__builtin_aarch64_rev16ll", uint64_fntype
,
1920 aarch64_builtin_decls
[AARCH64_RBIT
]
1921 = aarch64_general_add_builtin ("__builtin_aarch64_rbit", uint32_fntype
,
1923 aarch64_builtin_decls
[AARCH64_RBITL
]
1924 = aarch64_general_add_builtin ("__builtin_aarch64_rbitl", ulong_fntype
,
1926 aarch64_builtin_decls
[AARCH64_RBITLL
]
1927 = aarch64_general_add_builtin ("__builtin_aarch64_rbitll", uint64_fntype
,
1931 /* Implement #pragma GCC aarch64 "arm_acle.h". */
1933 handle_arm_acle_h (void)
1936 aarch64_init_ls64_builtins ();
1939 /* Initialize fpsr fpcr getters and setters. */
1942 aarch64_init_fpsr_fpcr_builtins (void)
1945 = build_function_type_list (void_type_node
, unsigned_type_node
, NULL
);
1947 = build_function_type_list (unsigned_type_node
, NULL
);
1949 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR
]
1950 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr",
1952 AARCH64_BUILTIN_GET_FPCR
);
1953 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR
]
1954 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr",
1956 AARCH64_BUILTIN_SET_FPCR
);
1957 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR
]
1958 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr",
1960 AARCH64_BUILTIN_GET_FPSR
);
1961 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR
]
1962 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr",
1964 AARCH64_BUILTIN_SET_FPSR
);
1967 = build_function_type_list (void_type_node
, long_long_unsigned_type_node
,
1970 = build_function_type_list (long_long_unsigned_type_node
, NULL
);
1972 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR64
]
1973 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr64",
1975 AARCH64_BUILTIN_GET_FPCR64
);
1976 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR64
]
1977 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr64",
1979 AARCH64_BUILTIN_SET_FPCR64
);
1980 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR64
]
1981 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr64",
1983 AARCH64_BUILTIN_GET_FPSR64
);
1984 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR64
]
1985 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr64",
1987 AARCH64_BUILTIN_SET_FPSR64
);
1990 /* Initialize all builtins in the AARCH64_BUILTIN_GENERAL group. */
1993 aarch64_general_init_builtins (void)
1995 aarch64_init_fpsr_fpcr_builtins ();
1997 aarch64_init_fp16_types ();
1999 aarch64_init_bf16_types ();
2002 aarch64_simd_switcher simd
;
2003 aarch64_init_simd_builtins ();
2006 aarch64_init_crc32_builtins ();
2007 aarch64_init_builtin_rsqrt ();
2008 aarch64_init_rng_builtins ();
2009 aarch64_init_data_intrinsics ();
2012 = build_function_type_list (intSI_type_node
, double_type_node
, NULL
);
2013 aarch64_builtin_decls
[AARCH64_JSCVT
]
2014 = aarch64_general_add_builtin ("__builtin_aarch64_jcvtzs", ftype_jcvt
,
2017 /* Initialize pointer authentication builtins which are backed by instructions
2018 in NOP encoding space.
2020 NOTE: these builtins are supposed to be used by libgcc unwinder only, as
2021 there is no support on return address signing under ILP32, we don't
2024 aarch64_init_pauth_hint_builtins ();
2027 aarch64_init_tme_builtins ();
2030 aarch64_init_memtag_builtins ();
2033 /* Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group. */
2035 aarch64_general_builtin_decl (unsigned code
, bool)
2037 if (code
>= AARCH64_BUILTIN_MAX
)
2038 return error_mark_node
;
2040 return aarch64_builtin_decls
[code
];
2045 SIMD_ARG_COPY_TO_REG
,
2047 SIMD_ARG_LANE_INDEX
,
2048 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
,
2049 SIMD_ARG_LANE_PAIR_INDEX
,
2050 SIMD_ARG_LANE_QUADTUP_INDEX
,
2056 aarch64_simd_expand_args (rtx target
, int icode
, int have_retval
,
2057 tree exp
, builtin_simd_arg
*args
,
2058 machine_mode builtin_mode
)
2061 rtx op
[SIMD_MAX_BUILTIN_ARGS
+ 1]; /* First element for result operand. */
2066 machine_mode tmode
= insn_data
[icode
].operand
[0].mode
;
2068 || GET_MODE (target
) != tmode
2069 || !(*insn_data
[icode
].operand
[0].predicate
) (target
, tmode
))
2070 target
= gen_reg_rtx (tmode
);
2076 builtin_simd_arg thisarg
= args
[opc
- have_retval
];
2078 if (thisarg
== SIMD_ARG_STOP
)
2082 tree arg
= CALL_EXPR_ARG (exp
, opc
- have_retval
);
2083 machine_mode mode
= insn_data
[icode
].operand
[opc
].mode
;
2084 op
[opc
] = expand_normal (arg
);
2088 case SIMD_ARG_COPY_TO_REG
:
2089 if (POINTER_TYPE_P (TREE_TYPE (arg
)))
2090 op
[opc
] = convert_memory_address (Pmode
, op
[opc
]);
2091 /*gcc_assert (GET_MODE (op[opc]) == mode); */
2092 if (!(*insn_data
[icode
].operand
[opc
].predicate
)
2094 op
[opc
] = copy_to_mode_reg (mode
, op
[opc
]);
2097 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
:
2098 gcc_assert (opc
> 1);
2099 if (CONST_INT_P (op
[opc
]))
2102 = GET_MODE_NUNITS (builtin_mode
).to_constant ();
2103 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
, exp
);
2104 /* Keep to GCC-vector-extension lane indices in the RTL. */
2105 op
[opc
] = aarch64_endian_lane_rtx (builtin_mode
,
2110 case SIMD_ARG_LANE_INDEX
:
2111 /* Must be a previous operand into which this is an index. */
2112 gcc_assert (opc
> 0);
2113 if (CONST_INT_P (op
[opc
]))
2115 machine_mode vmode
= insn_data
[icode
].operand
[opc
- 1].mode
;
2117 = GET_MODE_NUNITS (vmode
).to_constant ();
2118 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
, exp
);
2119 /* Keep to GCC-vector-extension lane indices in the RTL. */
2120 op
[opc
] = aarch64_endian_lane_rtx (vmode
, INTVAL (op
[opc
]));
2122 /* If the lane index isn't a constant then error out. */
2125 case SIMD_ARG_LANE_PAIR_INDEX
:
2126 /* Must be a previous operand into which this is an index and
2127 index is restricted to nunits / 2. */
2128 gcc_assert (opc
> 0);
2129 if (CONST_INT_P (op
[opc
]))
2131 machine_mode vmode
= insn_data
[icode
].operand
[opc
- 1].mode
;
2133 = GET_MODE_NUNITS (vmode
).to_constant ();
2134 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
/ 2, exp
);
2135 /* Keep to GCC-vector-extension lane indices in the RTL. */
2136 int lane
= INTVAL (op
[opc
]);
2137 op
[opc
] = gen_int_mode (ENDIAN_LANE_N (nunits
/ 2, lane
),
2140 /* If the lane index isn't a constant then error out. */
2142 case SIMD_ARG_LANE_QUADTUP_INDEX
:
2143 /* Must be a previous operand into which this is an index and
2144 index is restricted to nunits / 4. */
2145 gcc_assert (opc
> 0);
2146 if (CONST_INT_P (op
[opc
]))
2148 machine_mode vmode
= insn_data
[icode
].operand
[opc
- 1].mode
;
2150 = GET_MODE_NUNITS (vmode
).to_constant ();
2151 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
/ 4, exp
);
2152 /* Keep to GCC-vector-extension lane indices in the RTL. */
2153 int lane
= INTVAL (op
[opc
]);
2154 op
[opc
] = gen_int_mode (ENDIAN_LANE_N (nunits
/ 4, lane
),
2157 /* If the lane index isn't a constant then error out. */
2159 case SIMD_ARG_CONSTANT
:
2161 if (!(*insn_data
[icode
].operand
[opc
].predicate
)
2164 error_at (EXPR_LOCATION (exp
),
2165 "argument %d must be a constant immediate",
2166 opc
+ 1 - have_retval
);
2182 pat
= GEN_FCN (icode
) (op
[0]);
2186 pat
= GEN_FCN (icode
) (op
[0], op
[1]);
2190 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2]);
2194 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3]);
2198 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3], op
[4]);
2202 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3], op
[4], op
[5]);
2217 /* Expand an AArch64 AdvSIMD builtin(intrinsic). */
2219 aarch64_simd_expand_builtin (int fcode
, tree exp
, rtx target
)
2221 if (fcode
== AARCH64_SIMD_BUILTIN_LANE_CHECK
)
2223 rtx totalsize
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2224 rtx elementsize
= expand_normal (CALL_EXPR_ARG (exp
, 1));
2225 if (CONST_INT_P (totalsize
) && CONST_INT_P (elementsize
)
2226 && UINTVAL (elementsize
) != 0
2227 && UINTVAL (totalsize
) != 0)
2229 rtx lane_idx
= expand_normal (CALL_EXPR_ARG (exp
, 2));
2230 if (CONST_INT_P (lane_idx
))
2231 aarch64_simd_lane_bounds (lane_idx
, 0,
2233 / UINTVAL (elementsize
),
2236 error_at (EXPR_LOCATION (exp
),
2237 "lane index must be a constant immediate");
2240 error_at (EXPR_LOCATION (exp
),
2241 "total size and element size must be a nonzero "
2242 "constant immediate");
2243 /* Don't generate any RTL. */
2246 aarch64_simd_builtin_datum
*d
=
2247 &aarch64_simd_builtin_data
[fcode
- AARCH64_SIMD_PATTERN_START
];
2248 enum insn_code icode
= d
->code
;
2249 builtin_simd_arg args
[SIMD_MAX_BUILTIN_ARGS
+ 1];
2250 int num_args
= insn_data
[d
->code
].n_operands
;
2254 is_void
= !!(d
->qualifiers
[0] & qualifier_void
);
2256 num_args
+= is_void
;
2258 for (k
= 1; k
< num_args
; k
++)
2260 /* We have four arrays of data, each indexed in a different fashion.
2261 qualifiers - element 0 always describes the function return type.
2262 operands - element 0 is either the operand for return value (if
2263 the function has a non-void return type) or the operand for the
2265 expr_args - element 0 always holds the first argument.
2266 args - element 0 is always used for the return type. */
2267 int qualifiers_k
= k
;
2268 int operands_k
= k
- is_void
;
2269 int expr_args_k
= k
- 1;
2271 if (d
->qualifiers
[qualifiers_k
] & qualifier_lane_index
)
2272 args
[k
] = SIMD_ARG_LANE_INDEX
;
2273 else if (d
->qualifiers
[qualifiers_k
] & qualifier_lane_pair_index
)
2274 args
[k
] = SIMD_ARG_LANE_PAIR_INDEX
;
2275 else if (d
->qualifiers
[qualifiers_k
] & qualifier_lane_quadtup_index
)
2276 args
[k
] = SIMD_ARG_LANE_QUADTUP_INDEX
;
2277 else if (d
->qualifiers
[qualifiers_k
] & qualifier_struct_load_store_lane_index
)
2278 args
[k
] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
;
2279 else if (d
->qualifiers
[qualifiers_k
] & qualifier_immediate
)
2280 args
[k
] = SIMD_ARG_CONSTANT
;
2281 else if (d
->qualifiers
[qualifiers_k
] & qualifier_maybe_immediate
)
2284 = expand_normal (CALL_EXPR_ARG (exp
,
2286 /* Handle constants only if the predicate allows it. */
2287 bool op_const_int_p
=
2289 && (*insn_data
[icode
].operand
[operands_k
].predicate
)
2290 (arg
, insn_data
[icode
].operand
[operands_k
].mode
));
2291 args
[k
] = op_const_int_p
? SIMD_ARG_CONSTANT
: SIMD_ARG_COPY_TO_REG
;
2294 args
[k
] = SIMD_ARG_COPY_TO_REG
;
2297 args
[k
] = SIMD_ARG_STOP
;
2299 /* The interface to aarch64_simd_expand_args expects a 0 if
2300 the function is void, and a 1 if it is not. */
2301 return aarch64_simd_expand_args
2302 (target
, icode
, !is_void
, exp
, &args
[1], d
->mode
);
2306 aarch64_crc32_expand_builtin (int fcode
, tree exp
, rtx target
)
2309 aarch64_crc_builtin_datum
*d
2310 = &aarch64_crc_builtin_data
[fcode
- (AARCH64_CRC32_BUILTIN_BASE
+ 1)];
2311 enum insn_code icode
= d
->icode
;
2312 tree arg0
= CALL_EXPR_ARG (exp
, 0);
2313 tree arg1
= CALL_EXPR_ARG (exp
, 1);
2314 rtx op0
= expand_normal (arg0
);
2315 rtx op1
= expand_normal (arg1
);
2316 machine_mode tmode
= insn_data
[icode
].operand
[0].mode
;
2317 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2318 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2321 || GET_MODE (target
) != tmode
2322 || ! (*insn_data
[icode
].operand
[0].predicate
) (target
, tmode
))
2323 target
= gen_reg_rtx (tmode
);
2325 gcc_assert ((GET_MODE (op0
) == mode0
|| GET_MODE (op0
) == VOIDmode
)
2326 && (GET_MODE (op1
) == mode1
|| GET_MODE (op1
) == VOIDmode
));
2328 if (! (*insn_data
[icode
].operand
[1].predicate
) (op0
, mode0
))
2329 op0
= copy_to_mode_reg (mode0
, op0
);
2330 if (! (*insn_data
[icode
].operand
[2].predicate
) (op1
, mode1
))
2331 op1
= copy_to_mode_reg (mode1
, op1
);
2333 pat
= GEN_FCN (icode
) (target
, op0
, op1
);
2341 /* Function to expand reciprocal square root builtins. */
2344 aarch64_expand_builtin_rsqrt (int fcode
, tree exp
, rtx target
)
2346 tree arg0
= CALL_EXPR_ARG (exp
, 0);
2347 rtx op0
= expand_normal (arg0
);
2349 rtx (*gen
) (rtx
, rtx
);
2353 case AARCH64_BUILTIN_RSQRT_DF
:
2356 case AARCH64_BUILTIN_RSQRT_SF
:
2359 case AARCH64_BUILTIN_RSQRT_V2DF
:
2360 gen
= gen_rsqrtv2df2
;
2362 case AARCH64_BUILTIN_RSQRT_V2SF
:
2363 gen
= gen_rsqrtv2sf2
;
2365 case AARCH64_BUILTIN_RSQRT_V4SF
:
2366 gen
= gen_rsqrtv4sf2
;
2368 default: gcc_unreachable ();
2372 target
= gen_reg_rtx (GET_MODE (op0
));
2374 emit_insn (gen (target
, op0
));
2379 /* Expand a FCMLA lane expression EXP with code FCODE and
2380 result going to TARGET if that is convenient. */
2383 aarch64_expand_fcmla_builtin (tree exp
, rtx target
, int fcode
)
2385 int bcode
= fcode
- AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE
- 1;
2386 aarch64_fcmla_laneq_builtin_datum
* d
2387 = &aarch64_fcmla_lane_builtin_data
[bcode
];
2388 machine_mode quadmode
= GET_MODE_2XWIDER_MODE (d
->mode
).require ();
2389 rtx op0
= force_reg (d
->mode
, expand_normal (CALL_EXPR_ARG (exp
, 0)));
2390 rtx op1
= force_reg (d
->mode
, expand_normal (CALL_EXPR_ARG (exp
, 1)));
2391 rtx op2
= force_reg (quadmode
, expand_normal (CALL_EXPR_ARG (exp
, 2)));
2392 tree tmp
= CALL_EXPR_ARG (exp
, 3);
2393 rtx lane_idx
= expand_expr (tmp
, NULL_RTX
, VOIDmode
, EXPAND_INITIALIZER
);
2395 /* Validate that the lane index is a constant. */
2396 if (!CONST_INT_P (lane_idx
))
2398 error_at (EXPR_LOCATION (exp
),
2399 "argument %d must be a constant immediate", 4);
2403 /* Validate that the index is within the expected range. */
2404 int nunits
= GET_MODE_NUNITS (quadmode
).to_constant ();
2405 aarch64_simd_lane_bounds (lane_idx
, 0, nunits
/ 2, exp
);
2407 /* Generate the correct register and mode. */
2408 int lane
= INTVAL (lane_idx
);
2410 if (lane
< nunits
/ 4)
2411 op2
= simplify_gen_subreg (d
->mode
, op2
, quadmode
,
2412 subreg_lowpart_offset (d
->mode
, quadmode
));
2415 /* Select the upper 64 bits, either a V2SF or V4HF, this however
2416 is quite messy, as the operation required even though simple
2417 doesn't have a simple RTL pattern, and seems it's quite hard to
2418 define using a single RTL pattern. The target generic version
2419 gen_highpart_mode generates code that isn't optimal. */
2420 rtx temp1
= gen_reg_rtx (d
->mode
);
2421 rtx temp2
= gen_reg_rtx (DImode
);
2422 temp1
= simplify_gen_subreg (d
->mode
, op2
, quadmode
,
2423 subreg_lowpart_offset (d
->mode
, quadmode
));
2424 temp1
= simplify_gen_subreg (V2DImode
, temp1
, d
->mode
, 0);
2425 if (BYTES_BIG_ENDIAN
)
2426 emit_insn (gen_aarch64_get_lanev2di (temp2
, temp1
, const0_rtx
));
2428 emit_insn (gen_aarch64_get_lanev2di (temp2
, temp1
, const1_rtx
));
2429 op2
= simplify_gen_subreg (d
->mode
, temp2
, GET_MODE (temp2
), 0);
2431 /* And recalculate the index. */
2435 /* Keep to GCC-vector-extension lane indices in the RTL, only nunits / 4
2436 (max nunits in range check) are valid. Which means only 0-1, so we
2437 only need to know the order in a V2mode. */
2438 lane_idx
= aarch64_endian_lane_rtx (V2DImode
, lane
);
2442 || GET_MODE (target
) != d
->mode
)
2443 target
= gen_reg_rtx (d
->mode
);
2448 pat
= GEN_FCN (d
->icode
) (target
, op0
, op1
, op2
, lane_idx
);
2450 pat
= GEN_FCN (d
->icode
) (target
, op0
, op1
, op2
);
2459 /* Function to expand an expression EXP which calls one of the Transactional
2460 Memory Extension (TME) builtins FCODE with the result going to TARGET. */
2462 aarch64_expand_builtin_tme (int fcode
, tree exp
, rtx target
)
2466 case AARCH64_TME_BUILTIN_TSTART
:
2467 target
= gen_reg_rtx (DImode
);
2468 emit_insn (GEN_FCN (CODE_FOR_tstart
) (target
));
2471 case AARCH64_TME_BUILTIN_TTEST
:
2472 target
= gen_reg_rtx (DImode
);
2473 emit_insn (GEN_FCN (CODE_FOR_ttest
) (target
));
2476 case AARCH64_TME_BUILTIN_TCOMMIT
:
2477 emit_insn (GEN_FCN (CODE_FOR_tcommit
) ());
2480 case AARCH64_TME_BUILTIN_TCANCEL
:
2482 tree arg0
= CALL_EXPR_ARG (exp
, 0);
2483 rtx op0
= expand_normal (arg0
);
2484 if (CONST_INT_P (op0
) && UINTVAL (op0
) <= 65536)
2485 emit_insn (GEN_FCN (CODE_FOR_tcancel
) (op0
));
2488 error_at (EXPR_LOCATION (exp
),
2489 "argument must be a 16-bit constant immediate");
2501 /* Function to expand an expression EXP which calls one of the Load/Store
2502 64 Byte extension (LS64) builtins FCODE with the result going to TARGET. */
2504 aarch64_expand_builtin_ls64 (int fcode
, tree exp
, rtx target
)
2506 expand_operand ops
[3];
2510 case AARCH64_LS64_BUILTIN_LD64B
:
2512 rtx op0
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2513 create_output_operand (&ops
[0], target
, V8DImode
);
2514 create_input_operand (&ops
[1], op0
, DImode
);
2515 expand_insn (CODE_FOR_ld64b
, 2, ops
);
2516 return ops
[0].value
;
2518 case AARCH64_LS64_BUILTIN_ST64B
:
2520 rtx op0
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2521 rtx op1
= expand_normal (CALL_EXPR_ARG (exp
, 1));
2522 create_input_operand (&ops
[0], op0
, DImode
);
2523 create_input_operand (&ops
[1], op1
, V8DImode
);
2524 expand_insn (CODE_FOR_st64b
, 2, ops
);
2527 case AARCH64_LS64_BUILTIN_ST64BV
:
2529 rtx op0
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2530 rtx op1
= expand_normal (CALL_EXPR_ARG (exp
, 1));
2531 create_output_operand (&ops
[0], target
, DImode
);
2532 create_input_operand (&ops
[1], op0
, DImode
);
2533 create_input_operand (&ops
[2], op1
, V8DImode
);
2534 expand_insn (CODE_FOR_st64bv
, 3, ops
);
2535 return ops
[0].value
;
2537 case AARCH64_LS64_BUILTIN_ST64BV0
:
2539 rtx op0
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2540 rtx op1
= expand_normal (CALL_EXPR_ARG (exp
, 1));
2541 create_output_operand (&ops
[0], target
, DImode
);
2542 create_input_operand (&ops
[1], op0
, DImode
);
2543 create_input_operand (&ops
[2], op1
, V8DImode
);
2544 expand_insn (CODE_FOR_st64bv0
, 3, ops
);
2545 return ops
[0].value
;
2552 /* Expand a random number builtin EXP with code FCODE, putting the result
2553 int TARGET. If IGNORE is true the return value is ignored. */
2556 aarch64_expand_rng_builtin (tree exp
, rtx target
, int fcode
, int ignore
)
2559 enum insn_code icode
;
2560 if (fcode
== AARCH64_BUILTIN_RNG_RNDR
)
2561 icode
= CODE_FOR_aarch64_rndr
;
2562 else if (fcode
== AARCH64_BUILTIN_RNG_RNDRRS
)
2563 icode
= CODE_FOR_aarch64_rndrrs
;
2567 rtx rand
= gen_reg_rtx (DImode
);
2568 pat
= GEN_FCN (icode
) (rand
);
2572 tree arg0
= CALL_EXPR_ARG (exp
, 0);
2573 rtx res_addr
= expand_normal (arg0
);
2574 res_addr
= convert_memory_address (Pmode
, res_addr
);
2575 rtx res_mem
= gen_rtx_MEM (DImode
, res_addr
);
2577 emit_move_insn (res_mem
, rand
);
2578 /* If the status result is unused don't generate the CSET code. */
2582 rtx cc_reg
= gen_rtx_REG (CC_Zmode
, CC_REGNUM
);
2583 rtx cmp_rtx
= gen_rtx_fmt_ee (EQ
, SImode
, cc_reg
, const0_rtx
);
2584 emit_insn (gen_aarch64_cstoresi (target
, cmp_rtx
, cc_reg
));
2588 /* Expand an expression EXP that calls a MEMTAG built-in FCODE
2589 with result going to TARGET. */
2591 aarch64_expand_builtin_memtag (int fcode
, tree exp
, rtx target
)
2595 error ("Memory Tagging Extension does not support %<-mabi=ilp32%>");
2600 enum insn_code icode
= aarch64_memtag_builtin_data
[fcode
-
2601 AARCH64_MEMTAG_BUILTIN_START
- 1].icode
;
2603 rtx op0
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2604 machine_mode mode0
= GET_MODE (op0
);
2605 op0
= force_reg (mode0
== VOIDmode
? DImode
: mode0
, op0
);
2606 op0
= convert_to_mode (DImode
, op0
, true);
2610 case AARCH64_MEMTAG_BUILTIN_IRG
:
2611 case AARCH64_MEMTAG_BUILTIN_GMI
:
2612 case AARCH64_MEMTAG_BUILTIN_SUBP
:
2613 case AARCH64_MEMTAG_BUILTIN_INC_TAG
:
2616 || GET_MODE (target
) != DImode
2617 || ! (*insn_data
[icode
].operand
[0].predicate
) (target
, DImode
))
2618 target
= gen_reg_rtx (DImode
);
2620 if (fcode
== AARCH64_MEMTAG_BUILTIN_INC_TAG
)
2622 rtx op1
= expand_normal (CALL_EXPR_ARG (exp
, 1));
2624 if ((*insn_data
[icode
].operand
[3].predicate
) (op1
, QImode
))
2626 pat
= GEN_FCN (icode
) (target
, op0
, const0_rtx
, op1
);
2629 error_at (EXPR_LOCATION (exp
),
2630 "argument %d must be a constant immediate "
2631 "in range [0,15]", 2);
2636 rtx op1
= expand_normal (CALL_EXPR_ARG (exp
, 1));
2637 machine_mode mode1
= GET_MODE (op1
);
2638 op1
= force_reg (mode1
== VOIDmode
? DImode
: mode1
, op1
);
2639 op1
= convert_to_mode (DImode
, op1
, true);
2640 pat
= GEN_FCN (icode
) (target
, op0
, op1
);
2644 case AARCH64_MEMTAG_BUILTIN_GET_TAG
:
2646 pat
= GEN_FCN (icode
) (target
, op0
, const0_rtx
);
2648 case AARCH64_MEMTAG_BUILTIN_SET_TAG
:
2649 pat
= GEN_FCN (icode
) (op0
, op0
, const0_rtx
);
2662 /* Function to expand an expression EXP which calls one of the ACLE Data
2663 Intrinsic builtins FCODE with the result going to TARGET. */
2665 aarch64_expand_builtin_data_intrinsic (unsigned int fcode
, tree exp
, rtx target
)
2667 expand_operand ops
[2];
2668 machine_mode mode
= TYPE_MODE (TREE_TYPE (exp
));
2669 create_output_operand (&ops
[0], target
, mode
);
2670 create_input_operand (&ops
[1], expand_normal (CALL_EXPR_ARG (exp
, 0)), mode
);
2671 enum insn_code icode
;
2676 case AARCH64_REV16L
:
2677 case AARCH64_REV16LL
:
2678 icode
= code_for_aarch64_rev16 (mode
);
2682 case AARCH64_RBITLL
:
2683 icode
= code_for_aarch64_rbit (mode
);
2689 expand_insn (icode
, 2, ops
);
2690 return ops
[0].value
;
2693 /* Expand an expression EXP as fpsr or fpcr setter (depending on
2694 UNSPEC) using MODE. */
2696 aarch64_expand_fpsr_fpcr_setter (int unspec
, machine_mode mode
, tree exp
)
2698 tree arg
= CALL_EXPR_ARG (exp
, 0);
2699 rtx op
= force_reg (mode
, expand_normal (arg
));
2700 emit_insn (gen_aarch64_set (unspec
, mode
, op
));
2703 /* Expand a fpsr or fpcr getter (depending on UNSPEC) using MODE.
2704 Return the target. */
2706 aarch64_expand_fpsr_fpcr_getter (enum insn_code icode
, machine_mode mode
,
2710 create_output_operand (&op
, target
, mode
);
2711 expand_insn (icode
, 1, &op
);
2715 /* Expand an expression EXP that calls built-in function FCODE,
2716 with result going to TARGET if that's convenient. IGNORE is true
2717 if the result of the builtin is ignored. */
2719 aarch64_general_expand_builtin (unsigned int fcode
, tree exp
, rtx target
,
2728 case AARCH64_BUILTIN_GET_FPCR
:
2729 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrsi
,
2731 case AARCH64_BUILTIN_SET_FPCR
:
2732 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR
, SImode
, exp
);
2734 case AARCH64_BUILTIN_GET_FPSR
:
2735 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrsi
,
2737 case AARCH64_BUILTIN_SET_FPSR
:
2738 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR
, SImode
, exp
);
2740 case AARCH64_BUILTIN_GET_FPCR64
:
2741 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrdi
,
2743 case AARCH64_BUILTIN_SET_FPCR64
:
2744 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR
, DImode
, exp
);
2746 case AARCH64_BUILTIN_GET_FPSR64
:
2747 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrdi
,
2749 case AARCH64_BUILTIN_SET_FPSR64
:
2750 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR
, DImode
, exp
);
2752 case AARCH64_PAUTH_BUILTIN_AUTIA1716
:
2753 case AARCH64_PAUTH_BUILTIN_PACIA1716
:
2754 case AARCH64_PAUTH_BUILTIN_AUTIB1716
:
2755 case AARCH64_PAUTH_BUILTIN_PACIB1716
:
2756 case AARCH64_PAUTH_BUILTIN_XPACLRI
:
2757 arg0
= CALL_EXPR_ARG (exp
, 0);
2758 op0
= force_reg (Pmode
, expand_normal (arg0
));
2760 if (fcode
== AARCH64_PAUTH_BUILTIN_XPACLRI
)
2762 rtx lr
= gen_rtx_REG (Pmode
, R30_REGNUM
);
2763 icode
= CODE_FOR_xpaclri
;
2764 emit_move_insn (lr
, op0
);
2765 emit_insn (GEN_FCN (icode
) ());
2770 tree arg1
= CALL_EXPR_ARG (exp
, 1);
2771 rtx op1
= force_reg (Pmode
, expand_normal (arg1
));
2774 case AARCH64_PAUTH_BUILTIN_AUTIA1716
:
2775 icode
= CODE_FOR_autia1716
;
2777 case AARCH64_PAUTH_BUILTIN_AUTIB1716
:
2778 icode
= CODE_FOR_autib1716
;
2780 case AARCH64_PAUTH_BUILTIN_PACIA1716
:
2781 icode
= CODE_FOR_pacia1716
;
2783 case AARCH64_PAUTH_BUILTIN_PACIB1716
:
2784 icode
= CODE_FOR_pacib1716
;
2791 rtx x16_reg
= gen_rtx_REG (Pmode
, R16_REGNUM
);
2792 rtx x17_reg
= gen_rtx_REG (Pmode
, R17_REGNUM
);
2793 emit_move_insn (x17_reg
, op0
);
2794 emit_move_insn (x16_reg
, op1
);
2795 emit_insn (GEN_FCN (icode
) ());
2801 expand_operand ops
[2];
2802 create_output_operand (&ops
[0], target
, SImode
);
2803 op0
= expand_normal (CALL_EXPR_ARG (exp
, 0));
2804 create_input_operand (&ops
[1], op0
, DFmode
);
2805 expand_insn (CODE_FOR_aarch64_fjcvtzs
, 2, ops
);
2806 return ops
[0].value
;
2809 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF
:
2810 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF
:
2811 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF
:
2812 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF
:
2813 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF
:
2814 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF
:
2815 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF
:
2816 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF
:
2817 return aarch64_expand_fcmla_builtin (exp
, target
, fcode
);
2818 case AARCH64_BUILTIN_RNG_RNDR
:
2819 case AARCH64_BUILTIN_RNG_RNDRRS
:
2820 return aarch64_expand_rng_builtin (exp
, target
, fcode
, ignore
);
2823 if (fcode
>= AARCH64_SIMD_BUILTIN_BASE
&& fcode
<= AARCH64_SIMD_BUILTIN_MAX
)
2824 return aarch64_simd_expand_builtin (fcode
, exp
, target
);
2825 else if (fcode
>= AARCH64_CRC32_BUILTIN_BASE
&& fcode
<= AARCH64_CRC32_BUILTIN_MAX
)
2826 return aarch64_crc32_expand_builtin (fcode
, exp
, target
);
2828 if (fcode
== AARCH64_BUILTIN_RSQRT_DF
2829 || fcode
== AARCH64_BUILTIN_RSQRT_SF
2830 || fcode
== AARCH64_BUILTIN_RSQRT_V2DF
2831 || fcode
== AARCH64_BUILTIN_RSQRT_V2SF
2832 || fcode
== AARCH64_BUILTIN_RSQRT_V4SF
)
2833 return aarch64_expand_builtin_rsqrt (fcode
, exp
, target
);
2835 if (fcode
== AARCH64_TME_BUILTIN_TSTART
2836 || fcode
== AARCH64_TME_BUILTIN_TCOMMIT
2837 || fcode
== AARCH64_TME_BUILTIN_TTEST
2838 || fcode
== AARCH64_TME_BUILTIN_TCANCEL
)
2839 return aarch64_expand_builtin_tme (fcode
, exp
, target
);
2841 if (fcode
== AARCH64_LS64_BUILTIN_LD64B
2842 || fcode
== AARCH64_LS64_BUILTIN_ST64B
2843 || fcode
== AARCH64_LS64_BUILTIN_ST64BV
2844 || fcode
== AARCH64_LS64_BUILTIN_ST64BV0
)
2845 return aarch64_expand_builtin_ls64 (fcode
, exp
, target
);
2847 if (fcode
>= AARCH64_MEMTAG_BUILTIN_START
2848 && fcode
<= AARCH64_MEMTAG_BUILTIN_END
)
2849 return aarch64_expand_builtin_memtag (fcode
, exp
, target
);
2850 if (fcode
>= AARCH64_REV16
2851 && fcode
<= AARCH64_RBITLL
)
2852 return aarch64_expand_builtin_data_intrinsic (fcode
, exp
, target
);
2857 /* Return builtin for reciprocal square root. */
2860 aarch64_general_builtin_rsqrt (unsigned int fn
)
2862 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df
)
2863 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V2DF
];
2864 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf
)
2865 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V2SF
];
2866 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf
)
2867 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V4SF
];
2871 /* Return true if the lane check can be removed as there is no
2872 error going to be emitted. */
2874 aarch64_fold_builtin_lane_check (tree arg0
, tree arg1
, tree arg2
)
2876 if (TREE_CODE (arg0
) != INTEGER_CST
)
2878 if (TREE_CODE (arg1
) != INTEGER_CST
)
2880 if (TREE_CODE (arg2
) != INTEGER_CST
)
2883 auto totalsize
= wi::to_widest (arg0
);
2884 auto elementsize
= wi::to_widest (arg1
);
2885 if (totalsize
== 0 || elementsize
== 0)
2887 auto lane
= wi::to_widest (arg2
);
2888 auto high
= wi::udiv_trunc (totalsize
, elementsize
);
2889 return wi::ltu_p (lane
, high
);
2893 #define VAR1(T, N, MAP, FLAG, A) \
2894 case AARCH64_SIMD_BUILTIN_##T##_##N##A:
2896 #undef VREINTERPRET_BUILTIN
2897 #define VREINTERPRET_BUILTIN(A, B, L) \
2898 case AARCH64_SIMD_BUILTIN_VREINTERPRET##L##_##A##_##B:
2901 /* Try to fold a call to the built-in function with subcode FCODE. The
2902 function is passed the N_ARGS arguments in ARGS and it returns a value
2903 of type TYPE. Return the new expression on success and NULL_TREE on
2906 aarch64_general_fold_builtin (unsigned int fcode
, tree type
,
2907 unsigned int n_args ATTRIBUTE_UNUSED
, tree
*args
)
2911 BUILTIN_VDQF (UNOP
, abs
, 2, ALL
)
2912 return fold_build1 (ABS_EXPR
, type
, args
[0]);
2913 VAR1 (UNOP
, floatv2si
, 2, ALL
, v2sf
)
2914 VAR1 (UNOP
, floatv4si
, 2, ALL
, v4sf
)
2915 VAR1 (UNOP
, floatv2di
, 2, ALL
, v2df
)
2916 return fold_build1 (FLOAT_EXPR
, type
, args
[0]);
2917 AARCH64_SIMD_VREINTERPRET_BUILTINS
2918 return fold_build1 (VIEW_CONVERT_EXPR
, type
, args
[0]);
2919 case AARCH64_SIMD_BUILTIN_LANE_CHECK
:
2920 gcc_assert (n_args
== 3);
2921 if (aarch64_fold_builtin_lane_check (args
[0], args
[1], args
[2]))
2931 enum aarch64_simd_type
2932 get_mem_type_for_load_store (unsigned int fcode
)
2936 VAR1 (LOAD1
, ld1
, 0, LOAD
, v8qi
)
2937 VAR1 (STORE1
, st1
, 0, STORE
, v8qi
)
2939 VAR1 (LOAD1
, ld1
, 0, LOAD
, v16qi
)
2940 VAR1 (STORE1
, st1
, 0, STORE
, v16qi
)
2942 VAR1 (LOAD1
, ld1
, 0, LOAD
, v4hi
)
2943 VAR1 (STORE1
, st1
, 0, STORE
, v4hi
)
2945 VAR1 (LOAD1
, ld1
, 0, LOAD
, v8hi
)
2946 VAR1 (STORE1
, st1
, 0, STORE
, v8hi
)
2948 VAR1 (LOAD1
, ld1
, 0, LOAD
, v2si
)
2949 VAR1 (STORE1
, st1
, 0, STORE
, v2si
)
2951 VAR1 (LOAD1
, ld1
, 0, LOAD
, v4si
)
2952 VAR1 (STORE1
, st1
, 0, STORE
, v4si
)
2954 VAR1 (LOAD1
, ld1
, 0, LOAD
, v2di
)
2955 VAR1 (STORE1
, st1
, 0, STORE
, v2di
)
2957 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v8qi
)
2958 VAR1 (STORE1_U
, st1
, 0, STORE
, v8qi
)
2960 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v16qi
)
2961 VAR1 (STORE1_U
, st1
, 0, STORE
, v16qi
)
2963 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v4hi
)
2964 VAR1 (STORE1_U
, st1
, 0, STORE
, v4hi
)
2966 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v8hi
)
2967 VAR1 (STORE1_U
, st1
, 0, STORE
, v8hi
)
2969 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v2si
)
2970 VAR1 (STORE1_U
, st1
, 0, STORE
, v2si
)
2972 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v4si
)
2973 VAR1 (STORE1_U
, st1
, 0, STORE
, v4si
)
2975 VAR1 (LOAD1_U
, ld1
, 0, LOAD
, v2di
)
2976 VAR1 (STORE1_U
, st1
, 0, STORE
, v2di
)
2978 VAR1 (LOAD1_P
, ld1
, 0, LOAD
, v8qi
)
2979 VAR1 (STORE1_P
, st1
, 0, STORE
, v8qi
)
2981 VAR1 (LOAD1_P
, ld1
, 0, LOAD
, v16qi
)
2982 VAR1 (STORE1_P
, st1
, 0, STORE
, v16qi
)
2984 VAR1 (LOAD1_P
, ld1
, 0, LOAD
, v4hi
)
2985 VAR1 (STORE1_P
, st1
, 0, STORE
, v4hi
)
2987 VAR1 (LOAD1_P
, ld1
, 0, LOAD
, v8hi
)
2988 VAR1 (STORE1_P
, st1
, 0, STORE
, v8hi
)
2990 VAR1 (LOAD1_P
, ld1
, 0, LOAD
, v2di
)
2991 VAR1 (STORE1_P
, st1
, 0, STORE
, v2di
)
2993 VAR1 (LOAD1
, ld1
, 0, LOAD
, v4hf
)
2994 VAR1 (STORE1
, st1
, 0, STORE
, v4hf
)
2996 VAR1 (LOAD1
, ld1
, 0, LOAD
, v8hf
)
2997 VAR1 (STORE1
, st1
, 0, STORE
, v8hf
)
2999 VAR1 (LOAD1
, ld1
, 0, LOAD
, v4bf
)
3000 VAR1 (STORE1
, st1
, 0, STORE
, v4bf
)
3001 return Bfloat16x4_t
;
3002 VAR1 (LOAD1
, ld1
, 0, LOAD
, v8bf
)
3003 VAR1 (STORE1
, st1
, 0, STORE
, v8bf
)
3004 return Bfloat16x8_t
;
3005 VAR1 (LOAD1
, ld1
, 0, LOAD
, v2sf
)
3006 VAR1 (STORE1
, st1
, 0, STORE
, v2sf
)
3008 VAR1 (LOAD1
, ld1
, 0, LOAD
, v4sf
)
3009 VAR1 (STORE1
, st1
, 0, STORE
, v4sf
)
3011 VAR1 (LOAD1
, ld1
, 0, LOAD
, v2df
)
3012 VAR1 (STORE1
, st1
, 0, STORE
, v2df
)
3020 /* We've seen a vector load from address ADDR. Record it in
3021 vector_load_decls, if appropriate. */
3023 aarch64_record_vector_load_arg (tree addr
)
3025 tree decl
= aarch64_vector_load_decl (addr
);
3028 if (!cfun
->machine
->vector_load_decls
)
3029 cfun
->machine
->vector_load_decls
= hash_set
<tree
>::create_ggc (31);
3030 cfun
->machine
->vector_load_decls
->add (decl
);
3033 /* Try to fold STMT, given that it's a call to the built-in function with
3034 subcode FCODE. Return the new statement on success and null on
3037 aarch64_general_gimple_fold_builtin (unsigned int fcode
, gcall
*stmt
,
3038 gimple_stmt_iterator
*gsi ATTRIBUTE_UNUSED
)
3040 gimple
*new_stmt
= NULL
;
3041 unsigned nargs
= gimple_call_num_args (stmt
);
3042 tree
*args
= (nargs
> 0
3043 ? gimple_call_arg_ptr (stmt
, 0)
3044 : &error_mark_node
);
3046 /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int
3047 and unsigned int; it will distinguish according to the types of
3048 the arguments to the __builtin. */
3051 BUILTIN_VALL (UNOP
, reduc_plus_scal_
, 10, ALL
)
3052 BUILTIN_VDQ_I (UNOPU
, reduc_plus_scal_
, 10, NONE
)
3053 new_stmt
= gimple_build_call_internal (IFN_REDUC_PLUS
,
3055 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
3058 /* Lower sqrt builtins to gimple/internal function sqrt. */
3059 BUILTIN_VHSDF_DF (UNOP
, sqrt
, 2, FP
)
3060 new_stmt
= gimple_build_call_internal (IFN_SQRT
,
3062 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
3065 BUILTIN_VDC (BINOP
, combine
, 0, AUTO_FP
)
3066 BUILTIN_VD_I (BINOPU
, combine
, 0, NONE
)
3067 BUILTIN_VDC_P (BINOPP
, combine
, 0, NONE
)
3069 tree first_part
, second_part
;
3070 if (BYTES_BIG_ENDIAN
)
3072 second_part
= args
[0];
3073 first_part
= args
[1];
3077 first_part
= args
[0];
3078 second_part
= args
[1];
3080 tree ret_type
= gimple_call_return_type (stmt
);
3081 tree ctor
= build_constructor_va (ret_type
, 2, NULL_TREE
, first_part
,
3082 NULL_TREE
, second_part
);
3083 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
), ctor
);
3087 /*lower store and load neon builtins to gimple. */
3088 BUILTIN_VALL_F16 (LOAD1
, ld1
, 0, LOAD
)
3089 BUILTIN_VDQ_I (LOAD1_U
, ld1
, 0, LOAD
)
3090 BUILTIN_VALLP_NO_DI (LOAD1_P
, ld1
, 0, LOAD
)
3091 /* Punt until after inlining, so that we stand more chance of
3092 recording something meaningful in vector_load_decls. */
3093 if (!cfun
->after_inlining
)
3095 aarch64_record_vector_load_arg (args
[0]);
3096 if (!BYTES_BIG_ENDIAN
)
3098 enum aarch64_simd_type mem_type
3099 = get_mem_type_for_load_store(fcode
);
3100 aarch64_simd_type_info simd_type
3101 = aarch64_simd_types
[mem_type
];
3102 tree elt_ptr_type
= build_pointer_type_for_mode (simd_type
.eltype
,
3104 tree zero
= build_zero_cst (elt_ptr_type
);
3105 /* Use element type alignment. */
3107 = build_aligned_type (simd_type
.itype
,
3108 TYPE_ALIGN (simd_type
.eltype
));
3110 = gimple_build_assign (gimple_get_lhs (stmt
),
3111 fold_build2 (MEM_REF
,
3114 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
3115 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
3119 BUILTIN_VALL_F16 (STORE1
, st1
, 0, STORE
)
3120 BUILTIN_VDQ_I (STORE1_U
, st1
, 0, STORE
)
3121 BUILTIN_VALLP_NO_DI (STORE1_P
, st1
, 0, STORE
)
3122 if (!BYTES_BIG_ENDIAN
)
3124 enum aarch64_simd_type mem_type
3125 = get_mem_type_for_load_store(fcode
);
3126 aarch64_simd_type_info simd_type
3127 = aarch64_simd_types
[mem_type
];
3128 tree elt_ptr_type
= build_pointer_type_for_mode (simd_type
.eltype
,
3130 tree zero
= build_zero_cst (elt_ptr_type
);
3131 /* Use element type alignment. */
3133 = build_aligned_type (simd_type
.itype
,
3134 TYPE_ALIGN (simd_type
.eltype
));
3136 = gimple_build_assign (fold_build2 (MEM_REF
, access_type
,
3139 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
3140 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
3144 BUILTIN_VDQIF (UNOP
, reduc_smax_scal_
, 10, ALL
)
3145 BUILTIN_VDQ_BHSI (UNOPU
, reduc_umax_scal_
, 10, ALL
)
3146 new_stmt
= gimple_build_call_internal (IFN_REDUC_MAX
,
3148 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
3150 BUILTIN_VDQIF (UNOP
, reduc_smin_scal_
, 10, ALL
)
3151 BUILTIN_VDQ_BHSI (UNOPU
, reduc_umin_scal_
, 10, ALL
)
3152 new_stmt
= gimple_build_call_internal (IFN_REDUC_MIN
,
3154 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
3156 BUILTIN_VSDQ_I_DI (BINOP
, ashl
, 3, NONE
)
3157 if (TREE_CODE (args
[1]) == INTEGER_CST
3158 && wi::ltu_p (wi::to_wide (args
[1]), element_precision (args
[0])))
3159 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3160 LSHIFT_EXPR
, args
[0], args
[1]);
3162 BUILTIN_VSDQ_I_DI (BINOP
, sshl
, 0, NONE
)
3163 BUILTIN_VSDQ_I_DI (BINOP_UUS
, ushl
, 0, NONE
)
3166 tree ctype
= TREE_TYPE (cst
);
3167 /* Left shifts can be both scalar or vector, e.g. uint64x1_t is
3168 treated as a scalar type not a vector one. */
3169 if ((cst
= uniform_integer_cst_p (cst
)) != NULL_TREE
)
3171 wide_int wcst
= wi::to_wide (cst
);
3172 tree unit_ty
= TREE_TYPE (cst
);
3174 wide_int abs_cst
= wi::abs (wcst
);
3175 if (wi::geu_p (abs_cst
, element_precision (args
[0])))
3178 if (wi::neg_p (wcst
, TYPE_SIGN (ctype
)))
3181 final_cst
= wide_int_to_tree (unit_ty
, abs_cst
);
3182 if (TREE_CODE (cst
) != INTEGER_CST
)
3183 final_cst
= build_uniform_cst (ctype
, final_cst
);
3185 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3186 RSHIFT_EXPR
, args
[0],
3190 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3191 LSHIFT_EXPR
, args
[0], args
[1]);
3195 BUILTIN_VDQ_I (SHIFTIMM
, ashr
, 3, NONE
)
3196 VAR1 (SHIFTIMM
, ashr_simd
, 0, NONE
, di
)
3197 BUILTIN_VDQ_I (USHIFTIMM
, lshr
, 3, NONE
)
3198 VAR1 (USHIFTIMM
, lshr_simd
, 0, NONE
, di
)
3199 if (TREE_CODE (args
[1]) == INTEGER_CST
3200 && wi::ltu_p (wi::to_wide (args
[1]), element_precision (args
[0])))
3201 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3202 RSHIFT_EXPR
, args
[0], args
[1]);
3204 BUILTIN_GPF (BINOP
, fmulx
, 0, ALL
)
3206 gcc_assert (nargs
== 2);
3207 bool a0_cst_p
= TREE_CODE (args
[0]) == REAL_CST
;
3208 bool a1_cst_p
= TREE_CODE (args
[1]) == REAL_CST
;
3209 if (a0_cst_p
|| a1_cst_p
)
3211 if (a0_cst_p
&& a1_cst_p
)
3213 tree t0
= TREE_TYPE (args
[0]);
3214 real_value a0
= (TREE_REAL_CST (args
[0]));
3215 real_value a1
= (TREE_REAL_CST (args
[1]));
3216 if (real_equal (&a1
, &dconst0
))
3218 /* According to real_equal (), +0 equals -0. */
3219 if (real_equal (&a0
, &dconst0
) && real_isinf (&a1
))
3221 real_value res
= dconst2
;
3222 res
.sign
= a0
.sign
^ a1
.sign
;
3223 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3225 build_real (t0
, res
));
3228 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3232 else /* a0_cst_p ^ a1_cst_p. */
3234 real_value const_part
= a0_cst_p
3235 ? TREE_REAL_CST (args
[0]) : TREE_REAL_CST (args
[1]);
3236 if (!real_equal (&const_part
, &dconst0
)
3237 && !real_isinf (&const_part
))
3238 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
3245 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
3246 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
3250 case AARCH64_SIMD_BUILTIN_LANE_CHECK
:
3251 if (aarch64_fold_builtin_lane_check (args
[0], args
[1], args
[2]))
3253 unlink_stmt_vdef (stmt
);
3254 release_defs (stmt
);
3255 new_stmt
= gimple_build_nop ();
3262 /* GIMPLE assign statements (unlike calls) require a non-null lhs. If we
3263 created an assign statement with a null lhs, then fix this by assigning
3264 to a new (and subsequently unused) variable. */
3265 if (new_stmt
&& is_gimple_assign (new_stmt
) && !gimple_assign_lhs (new_stmt
))
3267 tree new_lhs
= make_ssa_name (gimple_call_return_type (stmt
));
3268 gimple_assign_set_lhs (new_stmt
, new_lhs
);
3275 aarch64_atomic_assign_expand_fenv (tree
*hold
, tree
*clear
, tree
*update
)
3277 const unsigned AARCH64_FE_INVALID
= 1;
3278 const unsigned AARCH64_FE_DIVBYZERO
= 2;
3279 const unsigned AARCH64_FE_OVERFLOW
= 4;
3280 const unsigned AARCH64_FE_UNDERFLOW
= 8;
3281 const unsigned AARCH64_FE_INEXACT
= 16;
3282 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT
= (AARCH64_FE_INVALID
3283 | AARCH64_FE_DIVBYZERO
3284 | AARCH64_FE_OVERFLOW
3285 | AARCH64_FE_UNDERFLOW
3286 | AARCH64_FE_INEXACT
);
3287 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT
= 8;
3288 tree fenv_cr
, fenv_sr
, get_fpcr
, set_fpcr
, mask_cr
, mask_sr
;
3289 tree ld_fenv_cr
, ld_fenv_sr
, masked_fenv_cr
, masked_fenv_sr
, hold_fnclex_cr
;
3290 tree hold_fnclex_sr
, new_fenv_var
, reload_fenv
, restore_fnenv
, get_fpsr
, set_fpsr
;
3291 tree update_call
, atomic_feraiseexcept
, hold_fnclex
, masked_fenv
, ld_fenv
;
3293 /* Generate the equivalence of :
3294 unsigned int fenv_cr;
3295 fenv_cr = __builtin_aarch64_get_fpcr ();
3297 unsigned int fenv_sr;
3298 fenv_sr = __builtin_aarch64_get_fpsr ();
3300 Now set all exceptions to non-stop
3301 unsigned int mask_cr
3302 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
3303 unsigned int masked_cr;
3304 masked_cr = fenv_cr & mask_cr;
3306 And clear all exception flags
3307 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
3308 unsigned int masked_cr;
3309 masked_sr = fenv_sr & mask_sr;
3311 __builtin_aarch64_set_cr (masked_cr);
3312 __builtin_aarch64_set_sr (masked_sr); */
3314 fenv_cr
= create_tmp_var_raw (unsigned_type_node
);
3315 fenv_sr
= create_tmp_var_raw (unsigned_type_node
);
3317 get_fpcr
= aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR
];
3318 set_fpcr
= aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR
];
3319 get_fpsr
= aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR
];
3320 set_fpsr
= aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR
];
3322 mask_cr
= build_int_cst (unsigned_type_node
,
3323 ~(AARCH64_FE_ALL_EXCEPT
<< AARCH64_FE_EXCEPT_SHIFT
));
3324 mask_sr
= build_int_cst (unsigned_type_node
,
3325 ~(AARCH64_FE_ALL_EXCEPT
));
3327 ld_fenv_cr
= build4 (TARGET_EXPR
, unsigned_type_node
,
3328 fenv_cr
, build_call_expr (get_fpcr
, 0),
3329 NULL_TREE
, NULL_TREE
);
3330 ld_fenv_sr
= build4 (TARGET_EXPR
, unsigned_type_node
,
3331 fenv_sr
, build_call_expr (get_fpsr
, 0),
3332 NULL_TREE
, NULL_TREE
);
3334 masked_fenv_cr
= build2 (BIT_AND_EXPR
, unsigned_type_node
, fenv_cr
, mask_cr
);
3335 masked_fenv_sr
= build2 (BIT_AND_EXPR
, unsigned_type_node
, fenv_sr
, mask_sr
);
3337 hold_fnclex_cr
= build_call_expr (set_fpcr
, 1, masked_fenv_cr
);
3338 hold_fnclex_sr
= build_call_expr (set_fpsr
, 1, masked_fenv_sr
);
3340 hold_fnclex
= build2 (COMPOUND_EXPR
, void_type_node
, hold_fnclex_cr
,
3342 masked_fenv
= build2 (COMPOUND_EXPR
, void_type_node
, masked_fenv_cr
,
3344 ld_fenv
= build2 (COMPOUND_EXPR
, void_type_node
, ld_fenv_cr
, ld_fenv_sr
);
3346 *hold
= build2 (COMPOUND_EXPR
, void_type_node
,
3347 build2 (COMPOUND_EXPR
, void_type_node
, masked_fenv
, ld_fenv
),
3350 /* Store the value of masked_fenv to clear the exceptions:
3351 __builtin_aarch64_set_fpsr (masked_fenv_sr); */
3353 *clear
= build_call_expr (set_fpsr
, 1, masked_fenv_sr
);
3355 /* Generate the equivalent of :
3356 unsigned int new_fenv_var;
3357 new_fenv_var = __builtin_aarch64_get_fpsr ();
3359 __builtin_aarch64_set_fpsr (fenv_sr);
3361 __atomic_feraiseexcept (new_fenv_var); */
3363 new_fenv_var
= create_tmp_var_raw (unsigned_type_node
);
3364 reload_fenv
= build4 (TARGET_EXPR
, unsigned_type_node
,
3365 new_fenv_var
, build_call_expr (get_fpsr
, 0),
3366 NULL_TREE
, NULL_TREE
);
3367 restore_fnenv
= build_call_expr (set_fpsr
, 1, fenv_sr
);
3368 atomic_feraiseexcept
= builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT
);
3369 update_call
= build_call_expr (atomic_feraiseexcept
, 1,
3370 fold_convert (integer_type_node
, new_fenv_var
));
3371 *update
= build2 (COMPOUND_EXPR
, void_type_node
,
3372 build2 (COMPOUND_EXPR
, void_type_node
,
3373 reload_fenv
, restore_fnenv
), update_call
);
3376 /* Resolve overloaded MEMTAG build-in functions. */
3377 #define AARCH64_BUILTIN_SUBCODE(F) \
3378 (DECL_MD_FUNCTION_CODE (F) >> AARCH64_BUILTIN_SHIFT)
3381 aarch64_resolve_overloaded_memtag (location_t loc
,
3382 tree fndecl
, void *pass_params
)
3384 vec
<tree
, va_gc
> *params
= static_cast<vec
<tree
, va_gc
> *> (pass_params
);
3385 unsigned param_num
= params
? params
->length() : 0;
3386 unsigned int fcode
= AARCH64_BUILTIN_SUBCODE (fndecl
);
3387 tree inittype
= aarch64_memtag_builtin_data
[
3388 fcode
- AARCH64_MEMTAG_BUILTIN_START
- 1].ftype
;
3389 unsigned arg_num
= list_length (TYPE_ARG_TYPES (inittype
)) - 1;
3391 if (param_num
!= arg_num
)
3393 TREE_TYPE (fndecl
) = inittype
;
3398 if (fcode
== AARCH64_MEMTAG_BUILTIN_SUBP
)
3400 tree t0
= TREE_TYPE ((*params
)[0]);
3401 tree t1
= TREE_TYPE ((*params
)[1]);
3403 if (t0
== error_mark_node
|| TREE_CODE (t0
) != POINTER_TYPE
)
3405 if (t1
== error_mark_node
|| TREE_CODE (t1
) != POINTER_TYPE
)
3408 if (TYPE_MODE (t0
) != DImode
)
3409 warning_at (loc
, 1, "expected 64-bit address but argument 1 is %d-bit",
3410 (int)tree_to_shwi (DECL_SIZE ((*params
)[0])));
3412 if (TYPE_MODE (t1
) != DImode
)
3413 warning_at (loc
, 1, "expected 64-bit address but argument 2 is %d-bit",
3414 (int)tree_to_shwi (DECL_SIZE ((*params
)[1])));
3416 retype
= build_function_type_list (ptrdiff_type_node
, t0
, t1
, NULL
);
3420 tree t0
= TREE_TYPE ((*params
)[0]);
3422 if (t0
== error_mark_node
|| TREE_CODE (t0
) != POINTER_TYPE
)
3424 TREE_TYPE (fndecl
) = inittype
;
3428 if (TYPE_MODE (t0
) != DImode
)
3429 warning_at (loc
, 1, "expected 64-bit address but argument 1 is %d-bit",
3430 (int)tree_to_shwi (DECL_SIZE ((*params
)[0])));
3434 case AARCH64_MEMTAG_BUILTIN_IRG
:
3435 retype
= build_function_type_list (t0
, t0
, uint64_type_node
, NULL
);
3437 case AARCH64_MEMTAG_BUILTIN_GMI
:
3438 retype
= build_function_type_list (uint64_type_node
, t0
,
3439 uint64_type_node
, NULL
);
3441 case AARCH64_MEMTAG_BUILTIN_INC_TAG
:
3442 retype
= build_function_type_list (t0
, t0
, unsigned_type_node
, NULL
);
3444 case AARCH64_MEMTAG_BUILTIN_SET_TAG
:
3445 retype
= build_function_type_list (void_type_node
, t0
, NULL
);
3447 case AARCH64_MEMTAG_BUILTIN_GET_TAG
:
3448 retype
= build_function_type_list (t0
, t0
, NULL
);
3455 if (!retype
|| retype
== error_mark_node
)
3456 TREE_TYPE (fndecl
) = inittype
;
3458 TREE_TYPE (fndecl
) = retype
;
3463 /* Called at aarch64_resolve_overloaded_builtin in aarch64-c.cc. */
3465 aarch64_resolve_overloaded_builtin_general (location_t loc
, tree function
,
3468 unsigned int fcode
= AARCH64_BUILTIN_SUBCODE (function
);
3470 if (fcode
>= AARCH64_MEMTAG_BUILTIN_START
3471 && fcode
<= AARCH64_MEMTAG_BUILTIN_END
)
3472 return aarch64_resolve_overloaded_memtag(loc
, function
, pass_params
);
3477 #undef AARCH64_CHECK_BUILTIN_MODE
3478 #undef AARCH64_FIND_FRINT_VARIANT
3497 #include "gt-aarch64-builtins.h"