]>
Commit | Line | Data |
---|---|---|
624d0f07 | 1 | /* ACLE support for AArch64 SVE |
8d9254fc | 2 | Copyright (C) 2018-2020 Free Software Foundation, Inc. |
624d0f07 RS |
3 | |
4 | This file is part of GCC. | |
5 | ||
6 | GCC is free software; you can redistribute it and/or modify it | |
7 | under the terms of the GNU General Public License as published by | |
8 | the Free Software Foundation; either version 3, or (at your option) | |
9 | any later version. | |
10 | ||
11 | GCC is distributed in the hope that it will be useful, but | |
12 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | General Public License for more details. | |
15 | ||
16 | You should have received a copy of the GNU General Public License | |
17 | along with GCC; see the file COPYING3. If not see | |
18 | <http://www.gnu.org/licenses/>. */ | |
19 | ||
20 | #define IN_TARGET_CODE 1 | |
21 | ||
22 | #include "config.h" | |
23 | #include "system.h" | |
24 | #include "coretypes.h" | |
25 | #include "tm.h" | |
26 | #include "tree.h" | |
27 | #include "rtl.h" | |
28 | #include "tm_p.h" | |
29 | #include "memmodel.h" | |
30 | #include "insn-codes.h" | |
31 | #include "optabs.h" | |
32 | #include "recog.h" | |
33 | #include "diagnostic.h" | |
34 | #include "expr.h" | |
35 | #include "basic-block.h" | |
36 | #include "function.h" | |
37 | #include "fold-const.h" | |
38 | #include "gimple.h" | |
39 | #include "gimple-iterator.h" | |
40 | #include "gimplify.h" | |
41 | #include "explow.h" | |
42 | #include "emit-rtl.h" | |
43 | #include "tree-vector-builder.h" | |
44 | #include "stor-layout.h" | |
45 | #include "regs.h" | |
46 | #include "alias.h" | |
47 | #include "gimple-fold.h" | |
48 | #include "langhooks.h" | |
49 | #include "stringpool.h" | |
683e93d1 | 50 | #include "attribs.h" |
624d0f07 RS |
51 | #include "aarch64-sve-builtins.h" |
52 | #include "aarch64-sve-builtins-base.h" | |
0a09a948 | 53 | #include "aarch64-sve-builtins-sve2.h" |
624d0f07 RS |
54 | #include "aarch64-sve-builtins-shapes.h" |
55 | ||
56 | namespace aarch64_sve { | |
57 | ||
58 | /* Static information about each single-predicate or single-vector | |
59 | ABI and ACLE type. */ | |
60 | struct vector_type_info | |
61 | { | |
62 | /* The name of the type as declared by arm_sve.h. */ | |
63 | const char *acle_name; | |
64 | ||
65 | /* The name of the type specified in AAPCS64. The type is always | |
66 | available under this name, even when arm_sve.h isn't included. */ | |
67 | const char *abi_name; | |
68 | ||
69 | /* The C++ mangling of ABI_NAME. */ | |
70 | const char *mangled_name; | |
71 | }; | |
72 | ||
73 | /* Describes a function decl. */ | |
74 | class GTY(()) registered_function | |
75 | { | |
76 | public: | |
77 | /* The ACLE function that the decl represents. */ | |
78 | function_instance instance GTY ((skip)); | |
79 | ||
80 | /* The decl itself. */ | |
81 | tree decl; | |
82 | ||
83 | /* The architecture extensions that the function requires, as a set of | |
84 | AARCH64_FL_* flags. */ | |
85 | uint64_t required_extensions; | |
86 | ||
87 | /* True if the decl represents an overloaded function that needs to be | |
88 | resolved by function_resolver. */ | |
89 | bool overloaded_p; | |
90 | }; | |
91 | ||
92 | /* Hash traits for registered_function. */ | |
93 | struct registered_function_hasher : nofree_ptr_hash <registered_function> | |
94 | { | |
95 | typedef function_instance compare_type; | |
96 | ||
97 | static hashval_t hash (value_type); | |
98 | static bool equal (value_type, const compare_type &); | |
99 | }; | |
100 | ||
101 | /* Information about each single-predicate or single-vector type. */ | |
102 | static CONSTEXPR const vector_type_info vector_types[] = { | |
103 | #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \ | |
dcb04335 | 104 | { #ACLE_NAME, #ABI_NAME, "u" #NCHARS #ABI_NAME }, |
624d0f07 RS |
105 | #include "aarch64-sve-builtins.def" |
106 | }; | |
107 | ||
108 | /* The function name suffix associated with each predication type. */ | |
109 | static const char *const pred_suffixes[NUM_PREDS + 1] = { | |
110 | "", | |
111 | "", | |
112 | "_m", | |
113 | "_x", | |
114 | "_z", | |
115 | "" | |
116 | }; | |
117 | ||
118 | /* Static information about each mode_suffix_index. */ | |
119 | CONSTEXPR const mode_suffix_info mode_suffixes[] = { | |
120 | #define VECTOR_TYPE_none NUM_VECTOR_TYPES | |
121 | #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) \ | |
122 | { "_" #NAME, VECTOR_TYPE_##BASE, VECTOR_TYPE_##DISPLACEMENT, UNITS_##UNITS }, | |
123 | #include "aarch64-sve-builtins.def" | |
124 | #undef VECTOR_TYPE_none | |
125 | { "", NUM_VECTOR_TYPES, NUM_VECTOR_TYPES, UNITS_none } | |
126 | }; | |
127 | ||
128 | /* Static information about each type_suffix_index. */ | |
129 | CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = { | |
130 | #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \ | |
131 | { "_" #NAME, \ | |
132 | VECTOR_TYPE_##ACLE_TYPE, \ | |
133 | TYPE_##CLASS, \ | |
134 | BITS, \ | |
135 | BITS / BITS_PER_UNIT, \ | |
136 | TYPE_##CLASS == TYPE_signed || TYPE_##CLASS == TYPE_unsigned, \ | |
137 | TYPE_##CLASS == TYPE_unsigned, \ | |
138 | TYPE_##CLASS == TYPE_float, \ | |
139 | TYPE_##CLASS == TYPE_bool, \ | |
140 | 0, \ | |
141 | MODE }, | |
142 | #include "aarch64-sve-builtins.def" | |
143 | { "", NUM_VECTOR_TYPES, TYPE_bool, 0, 0, false, false, false, false, | |
144 | 0, VOIDmode } | |
145 | }; | |
146 | ||
147 | /* Define a TYPES_<combination> macro for each combination of type | |
148 | suffixes that an ACLE function can have, where <combination> is the | |
149 | name used in DEF_SVE_FUNCTION entries. | |
150 | ||
151 | Use S (T) for single type suffix T and D (T1, T2) for a pair of type | |
152 | suffixes T1 and T2. Use commas to separate the suffixes. | |
153 | ||
154 | Although the order shouldn't matter, the convention is to sort the | |
155 | suffixes lexicographically after dividing suffixes into a type | |
156 | class ("b", "f", etc.) and a numerical bit count. */ | |
157 | ||
158 | /* _b8 _b16 _b32 _b64. */ | |
159 | #define TYPES_all_pred(S, D) \ | |
160 | S (b8), S (b16), S (b32), S (b64) | |
161 | ||
162 | /* _f16 _f32 _f64. */ | |
163 | #define TYPES_all_float(S, D) \ | |
164 | S (f16), S (f32), S (f64) | |
165 | ||
166 | /* _s8 _s16 _s32 _s64. */ | |
167 | #define TYPES_all_signed(S, D) \ | |
168 | S (s8), S (s16), S (s32), S (s64) | |
169 | ||
170 | /* _f16 _f32 _f64 | |
171 | _s8 _s16 _s32 _s64. */ | |
172 | #define TYPES_all_float_and_signed(S, D) \ | |
173 | TYPES_all_float (S, D), TYPES_all_signed (S, D) | |
174 | ||
175 | /* _u8 _u16 _u32 _u64. */ | |
176 | #define TYPES_all_unsigned(S, D) \ | |
177 | S (u8), S (u16), S (u32), S (u64) | |
178 | ||
179 | /* _s8 _s16 _s32 _s64 | |
180 | _u8 _u16 _u32 _u64. */ | |
181 | #define TYPES_all_integer(S, D) \ | |
182 | TYPES_all_signed (S, D), TYPES_all_unsigned (S, D) | |
183 | ||
184 | /* _f16 _f32 _f64 | |
185 | _s8 _s16 _s32 _s64 | |
186 | _u8 _u16 _u32 _u64. */ | |
02fcd8ac | 187 | #define TYPES_all_arith(S, D) \ |
624d0f07 RS |
188 | TYPES_all_float (S, D), TYPES_all_integer (S, D) |
189 | ||
02fcd8ac RS |
190 | /* _bf16 |
191 | _f16 _f32 _f64 | |
192 | _s8 _s16 _s32 _s64 | |
193 | _u8 _u16 _u32 _u64. */ | |
194 | #define TYPES_all_data(S, D) \ | |
195 | S (bf16), TYPES_all_arith (S, D) | |
196 | ||
624d0f07 RS |
197 | /* _b only. */ |
198 | #define TYPES_b(S, D) \ | |
199 | S (b) | |
200 | ||
0a09a948 RS |
201 | /* _u8. */ |
202 | #define TYPES_b_unsigned(S, D) \ | |
203 | S (u8) | |
204 | ||
205 | /* _s8 | |
206 | _u8. */ | |
207 | #define TYPES_b_integer(S, D) \ | |
208 | S (s8), TYPES_b_unsigned (S, D) | |
209 | ||
210 | /* _s8 _s16 | |
211 | _u8 _u16. */ | |
212 | #define TYPES_bh_integer(S, D) \ | |
213 | S (s8), S (s16), S (u8), S (u16) | |
214 | ||
215 | /* _u8 _u32. */ | |
216 | #define TYPES_bs_unsigned(S, D) \ | |
217 | S (u8), S (u32) | |
218 | ||
624d0f07 RS |
219 | /* _s8 _s16 _s32. */ |
220 | #define TYPES_bhs_signed(S, D) \ | |
221 | S (s8), S (s16), S (s32) | |
222 | ||
223 | /* _u8 _u16 _u32. */ | |
224 | #define TYPES_bhs_unsigned(S, D) \ | |
225 | S (u8), S (u16), S (u32) | |
226 | ||
227 | /* _s8 _s16 _s32 | |
228 | _u8 _u16 _u32. */ | |
229 | #define TYPES_bhs_integer(S, D) \ | |
230 | TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D) | |
231 | ||
232 | /* _s16 | |
233 | _u16. */ | |
234 | #define TYPES_h_integer(S, D) \ | |
235 | S (s16), S (u16) | |
236 | ||
0a09a948 RS |
237 | /* _s16 _s32. */ |
238 | #define TYPES_hs_signed(S, D) \ | |
239 | S (s16), S (s32) | |
240 | ||
241 | /* _s16 _s32 | |
242 | _u16 _u32. */ | |
243 | #define TYPES_hs_integer(S, D) \ | |
244 | TYPES_hs_signed (S, D), S (u16), S (u32) | |
245 | ||
624d0f07 RS |
246 | /* _f16 _f32. */ |
247 | #define TYPES_hs_float(S, D) \ | |
248 | S (f16), S (f32) | |
249 | ||
0a09a948 RS |
250 | /* _u16 _u64. */ |
251 | #define TYPES_hd_unsigned(S, D) \ | |
252 | S (u16), S (u64) | |
253 | ||
254 | /* _s16 _s32 _s64. */ | |
255 | #define TYPES_hsd_signed(S, D) \ | |
256 | S (s16), S (s32), S (s64) | |
257 | ||
624d0f07 RS |
258 | /* _s16 _s32 _s64 |
259 | _u16 _u32 _u64. */ | |
260 | #define TYPES_hsd_integer(S, D) \ | |
0a09a948 RS |
261 | TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64) |
262 | ||
36696774 RS |
263 | /* _f32. */ |
264 | #define TYPES_s_float(S, D) \ | |
265 | S (f32) | |
266 | ||
0a09a948 RS |
267 | /* _f32 |
268 | _s16 _s32 _s64 | |
269 | _u16 _u32 _u64. */ | |
270 | #define TYPES_s_float_hsd_integer(S, D) \ | |
36696774 | 271 | TYPES_s_float (S, D), TYPES_hsd_integer (S, D) |
0a09a948 RS |
272 | |
273 | /* _f32 | |
274 | _s32 _s64 | |
275 | _u32 _u64. */ | |
276 | #define TYPES_s_float_sd_integer(S, D) \ | |
36696774 RS |
277 | TYPES_s_float (S, D), TYPES_sd_integer (S, D) |
278 | ||
279 | /* _s32. */ | |
280 | #define TYPES_s_signed(S, D) \ | |
281 | S (s32) | |
0a09a948 RS |
282 | |
283 | /* _u32. */ | |
284 | #define TYPES_s_unsigned(S, D) \ | |
285 | S (u32) | |
624d0f07 RS |
286 | |
287 | /* _s32 _u32. */ | |
288 | #define TYPES_s_integer(S, D) \ | |
36696774 | 289 | TYPES_s_signed (S, D), TYPES_s_unsigned (S, D) |
0a09a948 RS |
290 | |
291 | /* _s32 _s64. */ | |
292 | #define TYPES_sd_signed(S, D) \ | |
293 | S (s32), S (s64) | |
294 | ||
295 | /* _u32 _u64. */ | |
296 | #define TYPES_sd_unsigned(S, D) \ | |
297 | S (u32), S (u64) | |
624d0f07 RS |
298 | |
299 | /* _s32 _s64 | |
300 | _u32 _u64. */ | |
301 | #define TYPES_sd_integer(S, D) \ | |
0a09a948 | 302 | TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D) |
624d0f07 RS |
303 | |
304 | /* _f32 _f64 | |
305 | _s32 _s64 | |
306 | _u32 _u64. */ | |
307 | #define TYPES_sd_data(S, D) \ | |
308 | S (f32), S (f64), TYPES_sd_integer (S, D) | |
309 | ||
310 | /* _f16 _f32 _f64 | |
311 | _s32 _s64 | |
312 | _u32 _u64. */ | |
313 | #define TYPES_all_float_and_sd_integer(S, D) \ | |
314 | TYPES_all_float (S, D), TYPES_sd_integer (S, D) | |
315 | ||
36696774 RS |
316 | /* _f64. */ |
317 | #define TYPES_d_float(S, D) \ | |
318 | S (f64) | |
319 | ||
0a09a948 RS |
320 | /* _u64. */ |
321 | #define TYPES_d_unsigned(S, D) \ | |
322 | S (u64) | |
323 | ||
624d0f07 RS |
324 | /* _s64 |
325 | _u64. */ | |
326 | #define TYPES_d_integer(S, D) \ | |
0a09a948 RS |
327 | S (s64), TYPES_d_unsigned (S, D) |
328 | ||
329 | /* _f64 | |
330 | _s64 | |
331 | _u64. */ | |
332 | #define TYPES_d_data(S, D) \ | |
36696774 | 333 | TYPES_d_float (S, D), TYPES_d_integer (S, D) |
624d0f07 RS |
334 | |
335 | /* All the type combinations allowed by svcvt. */ | |
336 | #define TYPES_cvt(S, D) \ | |
337 | D (f16, f32), D (f16, f64), \ | |
338 | D (f16, s16), D (f16, s32), D (f16, s64), \ | |
339 | D (f16, u16), D (f16, u32), D (f16, u64), \ | |
340 | \ | |
341 | D (f32, f16), D (f32, f64), \ | |
342 | D (f32, s32), D (f32, s64), \ | |
343 | D (f32, u32), D (f32, u64), \ | |
344 | \ | |
345 | D (f64, f16), D (f64, f32), \ | |
346 | D (f64, s32), D (f64, s64), \ | |
347 | D (f64, u32), D (f64, u64), \ | |
348 | \ | |
349 | D (s16, f16), \ | |
350 | D (s32, f16), D (s32, f32), D (s32, f64), \ | |
351 | D (s64, f16), D (s64, f32), D (s64, f64), \ | |
352 | \ | |
353 | D (u16, f16), \ | |
354 | D (u32, f16), D (u32, f32), D (u32, f64), \ | |
355 | D (u64, f16), D (u64, f32), D (u64, f64) | |
356 | ||
896dff99 RS |
357 | /* _bf16_f32. */ |
358 | #define TYPES_cvt_bfloat(S, D) \ | |
359 | D (bf16, f32) | |
360 | ||
0a09a948 RS |
361 | /* _f32_f16 |
362 | _f64_f32. */ | |
363 | #define TYPES_cvt_long(S, D) \ | |
364 | D (f32, f16), D (f64, f32) | |
365 | ||
366 | /* _f16_f32. */ | |
367 | #define TYPES_cvt_narrow_s(S, D) \ | |
368 | D (f32, f64) | |
369 | ||
370 | /* _f16_f32 | |
371 | _f32_f64. */ | |
372 | #define TYPES_cvt_narrow(S, D) \ | |
373 | D (f16, f32), TYPES_cvt_narrow_s (S, D) | |
374 | ||
624d0f07 RS |
375 | /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 } |
376 | { _u32 _u64 }. */ | |
377 | #define TYPES_inc_dec_n1(D, A) \ | |
378 | D (A, b8), D (A, b16), D (A, b32), D (A, b64) | |
379 | #define TYPES_inc_dec_n(S, D) \ | |
380 | TYPES_inc_dec_n1 (D, s32), \ | |
381 | TYPES_inc_dec_n1 (D, s64), \ | |
382 | TYPES_inc_dec_n1 (D, u32), \ | |
383 | TYPES_inc_dec_n1 (D, u64) | |
384 | ||
02fcd8ac RS |
385 | /* { _bf16 } { _bf16 } |
386 | { _f16 _f32 _f64 } { _f16 _f32 _f64 } | |
387 | { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 } | |
388 | { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }. */ | |
624d0f07 | 389 | #define TYPES_reinterpret1(D, A) \ |
02fcd8ac | 390 | D (A, bf16), \ |
624d0f07 RS |
391 | D (A, f16), D (A, f32), D (A, f64), \ |
392 | D (A, s8), D (A, s16), D (A, s32), D (A, s64), \ | |
393 | D (A, u8), D (A, u16), D (A, u32), D (A, u64) | |
394 | #define TYPES_reinterpret(S, D) \ | |
02fcd8ac | 395 | TYPES_reinterpret1 (D, bf16), \ |
624d0f07 RS |
396 | TYPES_reinterpret1 (D, f16), \ |
397 | TYPES_reinterpret1 (D, f32), \ | |
398 | TYPES_reinterpret1 (D, f64), \ | |
399 | TYPES_reinterpret1 (D, s8), \ | |
400 | TYPES_reinterpret1 (D, s16), \ | |
401 | TYPES_reinterpret1 (D, s32), \ | |
402 | TYPES_reinterpret1 (D, s64), \ | |
403 | TYPES_reinterpret1 (D, u8), \ | |
404 | TYPES_reinterpret1 (D, u16), \ | |
405 | TYPES_reinterpret1 (D, u32), \ | |
406 | TYPES_reinterpret1 (D, u64) | |
407 | ||
408 | /* { _b8 _b16 _b32 _b64 } x { _s32 _s64 } | |
409 | { _u32 _u64 } */ | |
410 | #define TYPES_while1(D, bn) \ | |
411 | D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64) | |
412 | #define TYPES_while(S, D) \ | |
413 | TYPES_while1 (D, b8), \ | |
414 | TYPES_while1 (D, b16), \ | |
415 | TYPES_while1 (D, b32), \ | |
416 | TYPES_while1 (D, b64) | |
417 | ||
418 | /* Describe a pair of type suffixes in which only the first is used. */ | |
419 | #define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES } | |
420 | ||
421 | /* Describe a pair of type suffixes in which both are used. */ | |
422 | #define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y } | |
423 | ||
424 | /* Create an array that can be used in aarch64-sve-builtins.def to | |
425 | select the type suffixes in TYPES_<NAME>. */ | |
426 | #define DEF_SVE_TYPES_ARRAY(NAME) \ | |
427 | static const type_suffix_pair types_##NAME[] = { \ | |
428 | TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \ | |
429 | { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \ | |
430 | } | |
431 | ||
432 | /* For functions that don't take any type suffixes. */ | |
433 | static const type_suffix_pair types_none[] = { | |
434 | { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }, | |
435 | { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } | |
436 | }; | |
437 | ||
438 | /* Create an array for each TYPES_<combination> macro above. */ | |
439 | DEF_SVE_TYPES_ARRAY (all_pred); | |
440 | DEF_SVE_TYPES_ARRAY (all_float); | |
441 | DEF_SVE_TYPES_ARRAY (all_signed); | |
442 | DEF_SVE_TYPES_ARRAY (all_float_and_signed); | |
443 | DEF_SVE_TYPES_ARRAY (all_unsigned); | |
444 | DEF_SVE_TYPES_ARRAY (all_integer); | |
02fcd8ac | 445 | DEF_SVE_TYPES_ARRAY (all_arith); |
624d0f07 RS |
446 | DEF_SVE_TYPES_ARRAY (all_data); |
447 | DEF_SVE_TYPES_ARRAY (b); | |
0a09a948 RS |
448 | DEF_SVE_TYPES_ARRAY (b_unsigned); |
449 | DEF_SVE_TYPES_ARRAY (b_integer); | |
450 | DEF_SVE_TYPES_ARRAY (bh_integer); | |
451 | DEF_SVE_TYPES_ARRAY (bs_unsigned); | |
624d0f07 RS |
452 | DEF_SVE_TYPES_ARRAY (bhs_signed); |
453 | DEF_SVE_TYPES_ARRAY (bhs_unsigned); | |
454 | DEF_SVE_TYPES_ARRAY (bhs_integer); | |
455 | DEF_SVE_TYPES_ARRAY (h_integer); | |
0a09a948 RS |
456 | DEF_SVE_TYPES_ARRAY (hs_signed); |
457 | DEF_SVE_TYPES_ARRAY (hs_integer); | |
624d0f07 | 458 | DEF_SVE_TYPES_ARRAY (hs_float); |
0a09a948 RS |
459 | DEF_SVE_TYPES_ARRAY (hd_unsigned); |
460 | DEF_SVE_TYPES_ARRAY (hsd_signed); | |
624d0f07 | 461 | DEF_SVE_TYPES_ARRAY (hsd_integer); |
36696774 | 462 | DEF_SVE_TYPES_ARRAY (s_float); |
0a09a948 RS |
463 | DEF_SVE_TYPES_ARRAY (s_float_hsd_integer); |
464 | DEF_SVE_TYPES_ARRAY (s_float_sd_integer); | |
36696774 | 465 | DEF_SVE_TYPES_ARRAY (s_signed); |
0a09a948 | 466 | DEF_SVE_TYPES_ARRAY (s_unsigned); |
624d0f07 | 467 | DEF_SVE_TYPES_ARRAY (s_integer); |
0a09a948 RS |
468 | DEF_SVE_TYPES_ARRAY (sd_signed); |
469 | DEF_SVE_TYPES_ARRAY (sd_unsigned); | |
624d0f07 RS |
470 | DEF_SVE_TYPES_ARRAY (sd_integer); |
471 | DEF_SVE_TYPES_ARRAY (sd_data); | |
472 | DEF_SVE_TYPES_ARRAY (all_float_and_sd_integer); | |
36696774 | 473 | DEF_SVE_TYPES_ARRAY (d_float); |
0a09a948 | 474 | DEF_SVE_TYPES_ARRAY (d_unsigned); |
624d0f07 | 475 | DEF_SVE_TYPES_ARRAY (d_integer); |
0a09a948 | 476 | DEF_SVE_TYPES_ARRAY (d_data); |
624d0f07 | 477 | DEF_SVE_TYPES_ARRAY (cvt); |
896dff99 | 478 | DEF_SVE_TYPES_ARRAY (cvt_bfloat); |
0a09a948 RS |
479 | DEF_SVE_TYPES_ARRAY (cvt_long); |
480 | DEF_SVE_TYPES_ARRAY (cvt_narrow_s); | |
481 | DEF_SVE_TYPES_ARRAY (cvt_narrow); | |
624d0f07 RS |
482 | DEF_SVE_TYPES_ARRAY (inc_dec_n); |
483 | DEF_SVE_TYPES_ARRAY (reinterpret); | |
484 | DEF_SVE_TYPES_ARRAY (while); | |
485 | ||
486 | /* Used by functions that have no governing predicate. */ | |
487 | static const predication_index preds_none[] = { PRED_none, NUM_PREDS }; | |
488 | ||
489 | /* Used by functions that have a governing predicate but do not have an | |
490 | explicit suffix. */ | |
491 | static const predication_index preds_implicit[] = { PRED_implicit, NUM_PREDS }; | |
492 | ||
0a09a948 RS |
493 | /* Used by functions that allow merging and "don't care" predication, |
494 | but are not suitable for predicated MOVPRFX. */ | |
495 | static const predication_index preds_mx[] = { | |
496 | PRED_m, PRED_x, NUM_PREDS | |
497 | }; | |
498 | ||
624d0f07 RS |
499 | /* Used by functions that allow merging, zeroing and "don't care" |
500 | predication. */ | |
501 | static const predication_index preds_mxz[] = { | |
502 | PRED_m, PRED_x, PRED_z, NUM_PREDS | |
503 | }; | |
504 | ||
505 | /* Used by functions that have the mxz predicated forms above, and in addition | |
506 | have an unpredicated form. */ | |
507 | static const predication_index preds_mxz_or_none[] = { | |
508 | PRED_m, PRED_x, PRED_z, PRED_none, NUM_PREDS | |
509 | }; | |
510 | ||
511 | /* Used by functions that allow merging and zeroing predication but have | |
512 | no "_x" form. */ | |
513 | static const predication_index preds_mz[] = { PRED_m, PRED_z, NUM_PREDS }; | |
514 | ||
515 | /* Used by functions that have an unpredicated form and a _z predicated | |
516 | form. */ | |
517 | static const predication_index preds_z_or_none[] = { | |
518 | PRED_z, PRED_none, NUM_PREDS | |
519 | }; | |
520 | ||
521 | /* Used by (mostly predicate) functions that only support "_z" predication. */ | |
522 | static const predication_index preds_z[] = { PRED_z, NUM_PREDS }; | |
523 | ||
524 | /* A list of all SVE ACLE functions. */ | |
525 | static CONSTEXPR const function_group_info function_groups[] = { | |
526 | #define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \ | |
527 | { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, preds_##PREDS, \ | |
528 | REQUIRED_EXTENSIONS | AARCH64_FL_SVE }, | |
529 | #include "aarch64-sve-builtins.def" | |
530 | }; | |
531 | ||
532 | /* The scalar type associated with each vector type. */ | |
533 | GTY(()) tree scalar_types[NUM_VECTOR_TYPES]; | |
534 | ||
535 | /* The single-predicate and single-vector types, with their built-in | |
536 | "__SV..._t" name. Allow an index of NUM_VECTOR_TYPES, which always | |
537 | yields a null tree. */ | |
538 | static GTY(()) tree abi_vector_types[NUM_VECTOR_TYPES + 1]; | |
539 | ||
540 | /* Same, but with the arm_sve.h "sv..._t" name. */ | |
541 | GTY(()) tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1]; | |
542 | ||
543 | /* The svpattern enum type. */ | |
544 | GTY(()) tree acle_svpattern; | |
545 | ||
546 | /* The svprfop enum type. */ | |
547 | GTY(()) tree acle_svprfop; | |
548 | ||
549 | /* The list of all registered function decls, indexed by code. */ | |
550 | static GTY(()) vec<registered_function *, va_gc> *registered_functions; | |
551 | ||
552 | /* All registered function decls, hashed on the function_instance | |
553 | that they implement. This is used for looking up implementations of | |
554 | overloaded functions. */ | |
555 | static hash_table<registered_function_hasher> *function_table; | |
556 | ||
557 | /* True if we've already complained about attempts to use functions | |
558 | when the required extension is disabled. */ | |
559 | static bool reported_missing_extension_p; | |
560 | ||
154ae7d4 FY |
561 | /* True if we've already complained about attempts to use functions |
562 | which require registers that are missing. */ | |
563 | static bool reported_missing_registers_p; | |
564 | ||
683e93d1 RS |
565 | /* Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors |
566 | and NUM_PR SVE predicates. MANGLED_NAME, if nonnull, is the ABI-defined | |
9ded41a3 | 567 | mangling of the type. ACLE_NAME is the <arm_sve.h> name of the type. */ |
683e93d1 RS |
568 | static void |
569 | add_sve_type_attribute (tree type, unsigned int num_zr, unsigned int num_pr, | |
9ded41a3 | 570 | const char *mangled_name, const char *acle_name) |
624d0f07 | 571 | { |
683e93d1 RS |
572 | tree mangled_name_tree |
573 | = (mangled_name ? get_identifier (mangled_name) : NULL_TREE); | |
574 | ||
9ded41a3 RS |
575 | tree value = tree_cons (NULL_TREE, get_identifier (acle_name), NULL_TREE); |
576 | value = tree_cons (NULL_TREE, mangled_name_tree, value); | |
683e93d1 RS |
577 | value = tree_cons (NULL_TREE, size_int (num_pr), value); |
578 | value = tree_cons (NULL_TREE, size_int (num_zr), value); | |
579 | TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE type"), value, | |
580 | TYPE_ATTRIBUTES (type)); | |
581 | } | |
582 | ||
583 | /* If TYPE is an ABI-defined SVE type, return its attribute descriptor, | |
584 | otherwise return null. */ | |
585 | static tree | |
586 | lookup_sve_type_attribute (const_tree type) | |
587 | { | |
588 | if (type == error_mark_node) | |
589 | return NULL_TREE; | |
590 | return lookup_attribute ("SVE type", TYPE_ATTRIBUTES (type)); | |
624d0f07 RS |
591 | } |
592 | ||
5002dae3 RS |
593 | /* Force TYPE to be a sizeless type. */ |
594 | static void | |
595 | make_type_sizeless (tree type) | |
596 | { | |
597 | TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE sizeless type"), | |
598 | NULL_TREE, TYPE_ATTRIBUTES (type)); | |
599 | } | |
600 | ||
601 | /* Return true if TYPE is a sizeless type. */ | |
602 | static bool | |
603 | sizeless_type_p (const_tree type) | |
604 | { | |
605 | if (type == error_mark_node) | |
606 | return NULL_TREE; | |
607 | return lookup_attribute ("SVE sizeless type", TYPE_ATTRIBUTES (type)); | |
608 | } | |
609 | ||
38e62001 RS |
610 | /* Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading |
611 | purposes. */ | |
612 | static bool | |
613 | matches_type_p (const_tree model_type, const_tree candidate) | |
614 | { | |
615 | if (VECTOR_TYPE_P (model_type)) | |
616 | { | |
617 | if (!VECTOR_TYPE_P (candidate) | |
618 | || maybe_ne (TYPE_VECTOR_SUBPARTS (model_type), | |
619 | TYPE_VECTOR_SUBPARTS (candidate)) | |
620 | || TYPE_MODE (model_type) != TYPE_MODE (candidate)) | |
621 | return false; | |
622 | ||
623 | model_type = TREE_TYPE (model_type); | |
624 | candidate = TREE_TYPE (candidate); | |
625 | } | |
626 | return (candidate != error_mark_node | |
627 | && TYPE_MAIN_VARIANT (model_type) == TYPE_MAIN_VARIANT (candidate)); | |
628 | } | |
629 | ||
624d0f07 RS |
630 | /* If TYPE is a valid SVE element type, return the corresponding type |
631 | suffix, otherwise return NUM_TYPE_SUFFIXES. */ | |
632 | static type_suffix_index | |
633 | find_type_suffix_for_scalar_type (const_tree type) | |
634 | { | |
635 | /* A linear search should be OK here, since the code isn't hot and | |
636 | the number of types is only small. */ | |
624d0f07 RS |
637 | for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i) |
638 | if (!type_suffixes[suffix_i].bool_p) | |
639 | { | |
640 | vector_type_index vector_i = type_suffixes[suffix_i].vector_type; | |
38e62001 | 641 | if (matches_type_p (scalar_types[vector_i], type)) |
624d0f07 RS |
642 | return type_suffix_index (suffix_i); |
643 | } | |
644 | return NUM_TYPE_SUFFIXES; | |
645 | } | |
646 | ||
647 | /* Report an error against LOCATION that the user has tried to use | |
648 | function FNDECL when extension EXTENSION is disabled. */ | |
649 | static void | |
650 | report_missing_extension (location_t location, tree fndecl, | |
651 | const char *extension) | |
652 | { | |
653 | /* Avoid reporting a slew of messages for a single oversight. */ | |
654 | if (reported_missing_extension_p) | |
655 | return; | |
656 | ||
657 | error_at (location, "ACLE function %qD requires ISA extension %qs", | |
658 | fndecl, extension); | |
659 | inform (location, "you can enable %qs using the command-line" | |
660 | " option %<-march%>, or by using the %<target%>" | |
661 | " attribute or pragma", extension); | |
662 | reported_missing_extension_p = true; | |
663 | } | |
664 | ||
154ae7d4 FY |
665 | /* Check whether the registers required by SVE function fndecl are available. |
666 | Report an error against LOCATION and return false if not. */ | |
667 | static bool | |
668 | check_required_registers (location_t location, tree fndecl) | |
669 | { | |
670 | /* Avoid reporting a slew of messages for a single oversight. */ | |
671 | if (reported_missing_registers_p) | |
672 | return false; | |
673 | ||
674 | if (TARGET_GENERAL_REGS_ONLY) | |
675 | { | |
676 | /* SVE registers are not usable when -mgeneral-regs-only option | |
677 | is specified. */ | |
678 | error_at (location, | |
679 | "ACLE function %qD is incompatible with the use of %qs", | |
680 | fndecl, "-mgeneral-regs-only"); | |
681 | reported_missing_registers_p = true; | |
682 | return false; | |
683 | } | |
684 | ||
685 | return true; | |
686 | } | |
687 | ||
624d0f07 RS |
688 | /* Check whether all the AARCH64_FL_* values in REQUIRED_EXTENSIONS are |
689 | enabled, given that those extensions are required for function FNDECL. | |
690 | Report an error against LOCATION if not. */ | |
691 | static bool | |
692 | check_required_extensions (location_t location, tree fndecl, | |
693 | uint64_t required_extensions) | |
694 | { | |
695 | uint64_t missing_extensions = required_extensions & ~aarch64_isa_flags; | |
696 | if (missing_extensions == 0) | |
154ae7d4 | 697 | return check_required_registers (location, fndecl); |
624d0f07 RS |
698 | |
699 | static const struct { uint64_t flag; const char *name; } extensions[] = { | |
700 | #define AARCH64_OPT_EXTENSION(EXT_NAME, FLAG_CANONICAL, FLAGS_ON, FLAGS_OFF, \ | |
701 | SYNTHETIC, FEATURE_STRING) \ | |
702 | { FLAG_CANONICAL, EXT_NAME }, | |
703 | #include "aarch64-option-extensions.def" | |
704 | }; | |
705 | ||
706 | for (unsigned int i = 0; i < ARRAY_SIZE (extensions); ++i) | |
707 | if (missing_extensions & extensions[i].flag) | |
708 | { | |
709 | report_missing_extension (location, fndecl, extensions[i].name); | |
710 | return false; | |
711 | } | |
712 | gcc_unreachable (); | |
713 | } | |
714 | ||
715 | /* Report that LOCATION has a call to FNDECL in which argument ARGNO | |
716 | was not an integer constant expression. ARGNO counts from zero. */ | |
717 | static void | |
718 | report_non_ice (location_t location, tree fndecl, unsigned int argno) | |
719 | { | |
720 | error_at (location, "argument %d of %qE must be an integer constant" | |
721 | " expression", argno + 1, fndecl); | |
722 | } | |
723 | ||
724 | /* Report that LOCATION has a call to FNDECL in which argument ARGNO has | |
725 | the value ACTUAL, whereas the function requires a value in the range | |
726 | [MIN, MAX]. ARGNO counts from zero. */ | |
727 | static void | |
728 | report_out_of_range (location_t location, tree fndecl, unsigned int argno, | |
729 | HOST_WIDE_INT actual, HOST_WIDE_INT min, | |
730 | HOST_WIDE_INT max) | |
731 | { | |
732 | error_at (location, "passing %wd to argument %d of %qE, which expects" | |
733 | " a value in the range [%wd, %wd]", actual, argno + 1, fndecl, | |
734 | min, max); | |
735 | } | |
736 | ||
737 | /* Report that LOCATION has a call to FNDECL in which argument ARGNO has | |
738 | the value ACTUAL, whereas the function requires either VALUE0 or | |
739 | VALUE1. ARGNO counts from zero. */ | |
740 | static void | |
741 | report_neither_nor (location_t location, tree fndecl, unsigned int argno, | |
742 | HOST_WIDE_INT actual, HOST_WIDE_INT value0, | |
743 | HOST_WIDE_INT value1) | |
744 | { | |
745 | error_at (location, "passing %wd to argument %d of %qE, which expects" | |
746 | " either %wd or %wd", actual, argno + 1, fndecl, value0, value1); | |
747 | } | |
748 | ||
749 | /* Report that LOCATION has a call to FNDECL in which argument ARGNO has | |
750 | the value ACTUAL, whereas the function requires one of VALUE0..3. | |
751 | ARGNO counts from zero. */ | |
752 | static void | |
753 | report_not_one_of (location_t location, tree fndecl, unsigned int argno, | |
754 | HOST_WIDE_INT actual, HOST_WIDE_INT value0, | |
755 | HOST_WIDE_INT value1, HOST_WIDE_INT value2, | |
756 | HOST_WIDE_INT value3) | |
757 | { | |
758 | error_at (location, "passing %wd to argument %d of %qE, which expects" | |
759 | " %wd, %wd, %wd or %wd", actual, argno + 1, fndecl, value0, value1, | |
760 | value2, value3); | |
761 | } | |
762 | ||
763 | /* Report that LOCATION has a call to FNDECL in which argument ARGNO has | |
764 | the value ACTUAL, whereas the function requires a valid value of | |
765 | enum type ENUMTYPE. ARGNO counts from zero. */ | |
766 | static void | |
767 | report_not_enum (location_t location, tree fndecl, unsigned int argno, | |
768 | HOST_WIDE_INT actual, tree enumtype) | |
769 | { | |
770 | error_at (location, "passing %wd to argument %d of %qE, which expects" | |
771 | " a valid %qT value", actual, argno + 1, fndecl, enumtype); | |
772 | } | |
773 | ||
774 | /* Return a hash code for a function_instance. */ | |
775 | hashval_t | |
776 | function_instance::hash () const | |
777 | { | |
778 | inchash::hash h; | |
779 | /* BASE uniquely determines BASE_NAME, so we don't need to hash both. */ | |
780 | h.add_ptr (base); | |
781 | h.add_ptr (shape); | |
782 | h.add_int (mode_suffix_id); | |
783 | h.add_int (type_suffix_ids[0]); | |
784 | h.add_int (type_suffix_ids[1]); | |
785 | h.add_int (pred); | |
786 | return h.end (); | |
787 | } | |
788 | ||
789 | /* Return a set of CP_* flags that describe what the function could do, | |
790 | taking the command-line flags into account. */ | |
791 | unsigned int | |
792 | function_instance::call_properties () const | |
793 | { | |
794 | unsigned int flags = base->call_properties (*this); | |
795 | ||
796 | /* -fno-trapping-math means that we can assume any FP exceptions | |
797 | are not user-visible. */ | |
798 | if (!flag_trapping_math) | |
799 | flags &= ~CP_RAISE_FP_EXCEPTIONS; | |
800 | ||
801 | return flags; | |
802 | } | |
803 | ||
804 | /* Return true if calls to the function could read some form of | |
805 | global state. */ | |
806 | bool | |
807 | function_instance::reads_global_state_p () const | |
808 | { | |
809 | unsigned int flags = call_properties (); | |
810 | ||
811 | /* Preserve any dependence on rounding mode, flush to zero mode, etc. | |
812 | There is currently no way of turning this off; in particular, | |
813 | -fno-rounding-math (which is the default) means that we should make | |
814 | the usual assumptions about rounding mode, which for intrinsics means | |
815 | acting as the instructions do. */ | |
816 | if (flags & CP_READ_FPCR) | |
817 | return true; | |
818 | ||
819 | /* Handle direct reads of global state. */ | |
820 | return flags & (CP_READ_MEMORY | CP_READ_FFR); | |
821 | } | |
822 | ||
823 | /* Return true if calls to the function could modify some form of | |
824 | global state. */ | |
825 | bool | |
826 | function_instance::modifies_global_state_p () const | |
827 | { | |
828 | unsigned int flags = call_properties (); | |
829 | ||
830 | /* Preserve any exception state written back to the FPCR, | |
831 | unless -fno-trapping-math says this is unnecessary. */ | |
832 | if (flags & CP_RAISE_FP_EXCEPTIONS) | |
833 | return true; | |
834 | ||
835 | /* Treat prefetches as modifying global state, since that's the | |
836 | only means we have of keeping them in their correct position. */ | |
837 | if (flags & CP_PREFETCH_MEMORY) | |
838 | return true; | |
839 | ||
840 | /* Handle direct modifications of global state. */ | |
841 | return flags & (CP_WRITE_MEMORY | CP_WRITE_FFR); | |
842 | } | |
843 | ||
844 | /* Return true if calls to the function could raise a signal. */ | |
845 | bool | |
846 | function_instance::could_trap_p () const | |
847 | { | |
848 | unsigned int flags = call_properties (); | |
849 | ||
850 | /* Handle functions that could raise SIGFPE. */ | |
851 | if (flags & CP_RAISE_FP_EXCEPTIONS) | |
852 | return true; | |
853 | ||
854 | /* Handle functions that could raise SIGBUS or SIGSEGV. */ | |
855 | if (flags & (CP_READ_MEMORY | CP_WRITE_MEMORY)) | |
856 | return true; | |
857 | ||
858 | return false; | |
859 | } | |
860 | ||
861 | inline hashval_t | |
862 | registered_function_hasher::hash (value_type value) | |
863 | { | |
864 | return value->instance.hash (); | |
865 | } | |
866 | ||
867 | inline bool | |
868 | registered_function_hasher::equal (value_type value, const compare_type &key) | |
869 | { | |
870 | return value->instance == key; | |
871 | } | |
872 | ||
873 | sve_switcher::sve_switcher () | |
874 | : m_old_isa_flags (aarch64_isa_flags) | |
875 | { | |
876 | /* Changing the ISA flags and have_regs_of_mode should be enough here. | |
877 | We shouldn't need to pay the compile-time cost of a full target | |
878 | switch. */ | |
879 | aarch64_isa_flags = (AARCH64_FL_FP | AARCH64_FL_SIMD | AARCH64_FL_F16 | |
880 | | AARCH64_FL_SVE); | |
881 | ||
b5cebc9a | 882 | m_old_maximum_field_alignment = maximum_field_alignment; |
883 | maximum_field_alignment = 0; | |
884 | ||
154ae7d4 FY |
885 | m_old_general_regs_only = TARGET_GENERAL_REGS_ONLY; |
886 | global_options.x_target_flags &= ~MASK_GENERAL_REGS_ONLY; | |
887 | ||
624d0f07 RS |
888 | memcpy (m_old_have_regs_of_mode, have_regs_of_mode, |
889 | sizeof (have_regs_of_mode)); | |
890 | for (int i = 0; i < NUM_MACHINE_MODES; ++i) | |
891 | if (aarch64_sve_mode_p ((machine_mode) i)) | |
892 | have_regs_of_mode[i] = true; | |
893 | } | |
894 | ||
895 | sve_switcher::~sve_switcher () | |
896 | { | |
897 | memcpy (have_regs_of_mode, m_old_have_regs_of_mode, | |
898 | sizeof (have_regs_of_mode)); | |
154ae7d4 FY |
899 | if (m_old_general_regs_only) |
900 | global_options.x_target_flags |= MASK_GENERAL_REGS_ONLY; | |
624d0f07 | 901 | aarch64_isa_flags = m_old_isa_flags; |
b5cebc9a | 902 | maximum_field_alignment = m_old_maximum_field_alignment; |
624d0f07 RS |
903 | } |
904 | ||
905 | function_builder::function_builder () | |
906 | { | |
907 | m_overload_type = build_function_type (void_type_node, void_list_node); | |
908 | m_direct_overloads = lang_GNU_CXX (); | |
909 | gcc_obstack_init (&m_string_obstack); | |
910 | } | |
911 | ||
912 | function_builder::~function_builder () | |
913 | { | |
914 | obstack_free (&m_string_obstack, NULL); | |
915 | } | |
916 | ||
917 | /* Add NAME to the end of the function name being built. */ | |
918 | void | |
919 | function_builder::append_name (const char *name) | |
920 | { | |
921 | obstack_grow (&m_string_obstack, name, strlen (name)); | |
922 | } | |
923 | ||
924 | /* Zero-terminate and complete the function name being built. */ | |
925 | char * | |
926 | function_builder::finish_name () | |
927 | { | |
928 | obstack_1grow (&m_string_obstack, 0); | |
929 | return (char *) obstack_finish (&m_string_obstack); | |
930 | } | |
931 | ||
932 | /* Return the overloaded or full function name for INSTANCE; OVERLOADED_P | |
933 | selects which. Allocate the string on m_string_obstack; the caller | |
934 | must use obstack_free to free it after use. */ | |
935 | char * | |
936 | function_builder::get_name (const function_instance &instance, | |
937 | bool overloaded_p) | |
938 | { | |
939 | append_name (instance.base_name); | |
940 | if (overloaded_p) | |
941 | switch (instance.displacement_units ()) | |
942 | { | |
943 | case UNITS_none: | |
944 | break; | |
945 | ||
946 | case UNITS_bytes: | |
947 | append_name ("_offset"); | |
948 | break; | |
949 | ||
950 | case UNITS_elements: | |
951 | append_name ("_index"); | |
952 | break; | |
953 | ||
954 | case UNITS_vectors: | |
955 | append_name ("_vnum"); | |
956 | break; | |
957 | } | |
958 | else | |
959 | append_name (instance.mode_suffix ().string); | |
960 | for (unsigned int i = 0; i < 2; ++i) | |
961 | if (!overloaded_p || instance.shape->explicit_type_suffix_p (i)) | |
962 | append_name (instance.type_suffix (i).string); | |
963 | append_name (pred_suffixes[instance.pred]); | |
964 | return finish_name (); | |
965 | } | |
966 | ||
967 | /* Add attribute NAME to ATTRS. */ | |
968 | static tree | |
969 | add_attribute (const char *name, tree attrs) | |
970 | { | |
971 | return tree_cons (get_identifier (name), NULL_TREE, attrs); | |
972 | } | |
973 | ||
974 | /* Return the appropriate function attributes for INSTANCE. */ | |
975 | tree | |
976 | function_builder::get_attributes (const function_instance &instance) | |
977 | { | |
978 | tree attrs = NULL_TREE; | |
979 | ||
980 | if (!instance.modifies_global_state_p ()) | |
981 | { | |
982 | if (instance.reads_global_state_p ()) | |
983 | attrs = add_attribute ("pure", attrs); | |
984 | else | |
985 | attrs = add_attribute ("const", attrs); | |
986 | } | |
987 | ||
988 | if (!flag_non_call_exceptions || !instance.could_trap_p ()) | |
989 | attrs = add_attribute ("nothrow", attrs); | |
990 | ||
991 | return add_attribute ("leaf", attrs); | |
992 | } | |
993 | ||
994 | /* Add a function called NAME with type FNTYPE and attributes ATTRS. | |
995 | INSTANCE describes what the function does and OVERLOADED_P indicates | |
996 | whether it is overloaded. REQUIRED_EXTENSIONS are the set of | |
997 | architecture extensions that the function requires. */ | |
998 | registered_function & | |
999 | function_builder::add_function (const function_instance &instance, | |
1000 | const char *name, tree fntype, tree attrs, | |
1001 | uint64_t required_extensions, | |
1002 | bool overloaded_p) | |
1003 | { | |
1004 | unsigned int code = vec_safe_length (registered_functions); | |
1005 | code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_SVE; | |
1006 | tree decl = simulate_builtin_function_decl (input_location, name, fntype, | |
1007 | code, NULL, attrs); | |
1008 | ||
1009 | registered_function &rfn = *ggc_alloc <registered_function> (); | |
1010 | rfn.instance = instance; | |
1011 | rfn.decl = decl; | |
1012 | rfn.required_extensions = required_extensions; | |
1013 | rfn.overloaded_p = overloaded_p; | |
1014 | vec_safe_push (registered_functions, &rfn); | |
1015 | ||
1016 | return rfn; | |
1017 | } | |
1018 | ||
1019 | /* Add a built-in function for INSTANCE, with the argument types given | |
1020 | by ARGUMENT_TYPES and the return type given by RETURN_TYPE. | |
1021 | REQUIRED_EXTENSIONS are the set of architecture extensions that the | |
1022 | function requires. FORCE_DIRECT_OVERLOADS is true if there is a | |
1023 | one-to-one mapping between "short" and "full" names, and if standard | |
1024 | overload resolution therefore isn't necessary. */ | |
1025 | void | |
1026 | function_builder::add_unique_function (const function_instance &instance, | |
1027 | tree return_type, | |
1028 | vec<tree> &argument_types, | |
1029 | uint64_t required_extensions, | |
1030 | bool force_direct_overloads) | |
1031 | { | |
1032 | /* Add the function under its full (unique) name. */ | |
1033 | char *name = get_name (instance, false); | |
1034 | tree fntype = build_function_type_array (return_type, | |
1035 | argument_types.length (), | |
1036 | argument_types.address ()); | |
1037 | tree attrs = get_attributes (instance); | |
1038 | registered_function &rfn = add_function (instance, name, fntype, attrs, | |
1039 | required_extensions, false); | |
1040 | ||
1041 | /* Enter the function into the hash table. */ | |
1042 | hashval_t hash = instance.hash (); | |
1043 | registered_function **rfn_slot | |
1044 | = function_table->find_slot_with_hash (instance, hash, INSERT); | |
1045 | gcc_assert (!*rfn_slot); | |
1046 | *rfn_slot = &rfn; | |
1047 | ||
1048 | /* Also add the function under its overloaded alias, if we want | |
1049 | a separate decl for each instance of an overloaded function. */ | |
1050 | if (m_direct_overloads || force_direct_overloads) | |
1051 | { | |
1052 | char *overload_name = get_name (instance, true); | |
1053 | if (strcmp (name, overload_name) != 0) | |
1054 | { | |
1055 | /* Attribute lists shouldn't be shared. */ | |
1056 | tree attrs = get_attributes (instance); | |
1057 | add_function (instance, overload_name, fntype, attrs, | |
1058 | required_extensions, false); | |
1059 | } | |
1060 | } | |
1061 | ||
1062 | obstack_free (&m_string_obstack, name); | |
1063 | } | |
1064 | ||
1065 | /* Add one function decl for INSTANCE, to be used with manual overload | |
1066 | resolution. REQUIRED_EXTENSIONS are the set of architecture extensions | |
1067 | that the function requires. | |
1068 | ||
0a09a948 RS |
1069 | For simplicity, deal with duplicate attempts to add the same function, |
1070 | including cases in which the new function requires more features than | |
1071 | the original one did. In that case we'll check whether the required | |
1072 | features are available as part of resolving the function to the | |
1073 | relevant unique function. */ | |
624d0f07 RS |
1074 | void |
1075 | function_builder::add_overloaded_function (const function_instance &instance, | |
1076 | uint64_t required_extensions) | |
1077 | { | |
1078 | char *name = get_name (instance, true); | |
1079 | if (registered_function **map_value = m_overload_names.get (name)) | |
1080 | gcc_assert ((*map_value)->instance == instance | |
0a09a948 RS |
1081 | && ((*map_value)->required_extensions |
1082 | & ~required_extensions) == 0); | |
624d0f07 RS |
1083 | else |
1084 | { | |
1085 | registered_function &rfn | |
1086 | = add_function (instance, name, m_overload_type, NULL_TREE, | |
1087 | required_extensions, true); | |
1088 | const char *permanent_name = IDENTIFIER_POINTER (DECL_NAME (rfn.decl)); | |
1089 | m_overload_names.put (permanent_name, &rfn); | |
1090 | } | |
1091 | obstack_free (&m_string_obstack, name); | |
1092 | } | |
1093 | ||
1094 | /* If we are using manual overload resolution, add one function decl | |
1095 | for each overloaded function in GROUP. Take the function base name | |
1096 | from GROUP and the mode from MODE. */ | |
1097 | void | |
1098 | function_builder::add_overloaded_functions (const function_group_info &group, | |
1099 | mode_suffix_index mode) | |
1100 | { | |
1101 | if (m_direct_overloads) | |
1102 | return; | |
1103 | ||
1104 | unsigned int explicit_type0 = (*group.shape)->explicit_type_suffix_p (0); | |
1105 | unsigned int explicit_type1 = (*group.shape)->explicit_type_suffix_p (1); | |
1106 | for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi) | |
1107 | { | |
1108 | if (!explicit_type0 && !explicit_type1) | |
1109 | { | |
1110 | /* Deal with the common case in which there is one overloaded | |
1111 | function for all type combinations. */ | |
1112 | function_instance instance (group.base_name, *group.base, | |
1113 | *group.shape, mode, types_none[0], | |
1114 | group.preds[pi]); | |
1115 | add_overloaded_function (instance, group.required_extensions); | |
1116 | } | |
1117 | else | |
1118 | for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES; | |
1119 | ++ti) | |
1120 | { | |
1121 | /* Stub out the types that are determined by overload | |
1122 | resolution. */ | |
1123 | type_suffix_pair types = { | |
1124 | explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES, | |
1125 | explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES | |
1126 | }; | |
1127 | function_instance instance (group.base_name, *group.base, | |
1128 | *group.shape, mode, types, | |
1129 | group.preds[pi]); | |
1130 | add_overloaded_function (instance, group.required_extensions); | |
1131 | } | |
1132 | } | |
1133 | } | |
1134 | ||
1135 | /* Register all the functions in GROUP. */ | |
1136 | void | |
1137 | function_builder::register_function_group (const function_group_info &group) | |
1138 | { | |
1139 | (*group.shape)->build (*this, group); | |
1140 | } | |
1141 | ||
1142 | function_call_info::function_call_info (location_t location_in, | |
1143 | const function_instance &instance_in, | |
1144 | tree fndecl_in) | |
1145 | : function_instance (instance_in), location (location_in), fndecl (fndecl_in) | |
1146 | { | |
1147 | } | |
1148 | ||
1149 | function_resolver::function_resolver (location_t location, | |
1150 | const function_instance &instance, | |
1151 | tree fndecl, vec<tree, va_gc> &arglist) | |
1152 | : function_call_info (location, instance, fndecl), m_arglist (arglist) | |
1153 | { | |
1154 | } | |
1155 | ||
1156 | /* Return the vector type associated with type suffix TYPE. */ | |
1157 | tree | |
1158 | function_resolver::get_vector_type (type_suffix_index type) | |
1159 | { | |
1160 | return acle_vector_types[0][type_suffixes[type].vector_type]; | |
1161 | } | |
1162 | ||
1163 | /* Return the <stdint.h> name associated with TYPE. Using the <stdint.h> | |
1164 | name should be more user-friendly than the underlying canonical type, | |
1165 | since it makes the signedness and bitwidth explicit. */ | |
1166 | const char * | |
1167 | function_resolver::get_scalar_type_name (type_suffix_index type) | |
1168 | { | |
1169 | return vector_types[type_suffixes[type].vector_type].acle_name + 2; | |
1170 | } | |
1171 | ||
1172 | /* Return the type of argument I, or error_mark_node if it isn't | |
1173 | well-formed. */ | |
1174 | tree | |
1175 | function_resolver::get_argument_type (unsigned int i) | |
1176 | { | |
1177 | tree arg = m_arglist[i]; | |
1178 | return arg == error_mark_node ? arg : TREE_TYPE (arg); | |
1179 | } | |
1180 | ||
1181 | /* Return true if argument I is some form of scalar value. */ | |
1182 | bool | |
1183 | function_resolver::scalar_argument_p (unsigned int i) | |
1184 | { | |
1185 | tree type = get_argument_type (i); | |
1186 | return (INTEGRAL_TYPE_P (type) | |
1187 | /* Allow pointer types, leaving the frontend to warn where | |
1188 | necessary. */ | |
1189 | || POINTER_TYPE_P (type) | |
1190 | || SCALAR_FLOAT_TYPE_P (type)); | |
1191 | } | |
1192 | ||
1193 | /* Report that the function has no form that takes type suffix TYPE. | |
1194 | Return error_mark_node. */ | |
1195 | tree | |
1196 | function_resolver::report_no_such_form (type_suffix_index type) | |
1197 | { | |
1198 | error_at (location, "%qE has no form that takes %qT arguments", | |
1199 | fndecl, get_vector_type (type)); | |
1200 | return error_mark_node; | |
1201 | } | |
1202 | ||
1203 | /* Silently check whether there is an instance of the function with the | |
1204 | mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1. | |
1205 | Return its function decl if so, otherwise return null. */ | |
1206 | tree | |
1207 | function_resolver::lookup_form (mode_suffix_index mode, | |
1208 | type_suffix_index type0, | |
1209 | type_suffix_index type1) | |
1210 | { | |
1211 | type_suffix_pair types = { type0, type1 }; | |
1212 | function_instance instance (base_name, base, shape, mode, types, pred); | |
1213 | registered_function *rfn | |
1214 | = function_table->find_with_hash (instance, instance.hash ()); | |
1215 | return rfn ? rfn->decl : NULL_TREE; | |
1216 | } | |
1217 | ||
1218 | /* Resolve the function to one with the mode suffix given by MODE and the | |
1219 | type suffixes given by TYPE0 and TYPE1. Return its function decl on | |
1220 | success, otherwise report an error and return error_mark_node. */ | |
1221 | tree | |
1222 | function_resolver::resolve_to (mode_suffix_index mode, | |
1223 | type_suffix_index type0, | |
1224 | type_suffix_index type1) | |
1225 | { | |
1226 | tree res = lookup_form (mode, type0, type1); | |
1227 | if (!res) | |
1228 | { | |
1229 | if (type1 == NUM_TYPE_SUFFIXES) | |
1230 | return report_no_such_form (type0); | |
1231 | if (type0 == type_suffix_ids[0]) | |
1232 | return report_no_such_form (type1); | |
1233 | /* To be filled in when we have other cases. */ | |
1234 | gcc_unreachable (); | |
1235 | } | |
1236 | return res; | |
1237 | } | |
1238 | ||
1239 | /* Require argument ARGNO to be a 32-bit or 64-bit scalar integer type. | |
1240 | Return the associated type suffix on success, otherwise report an | |
1241 | error and return NUM_TYPE_SUFFIXES. */ | |
1242 | type_suffix_index | |
1243 | function_resolver::infer_integer_scalar_type (unsigned int argno) | |
1244 | { | |
1245 | tree actual = get_argument_type (argno); | |
1246 | if (actual == error_mark_node) | |
1247 | return NUM_TYPE_SUFFIXES; | |
1248 | ||
1249 | /* Allow enums and booleans to decay to integers, for compatibility | |
1250 | with C++ overloading rules. */ | |
1251 | if (INTEGRAL_TYPE_P (actual)) | |
1252 | { | |
1253 | bool uns_p = TYPE_UNSIGNED (actual); | |
1254 | /* Honor the usual integer promotions, so that resolution works | |
1255 | in the same way as for C++. */ | |
1256 | if (TYPE_PRECISION (actual) < 32) | |
1257 | return TYPE_SUFFIX_s32; | |
1258 | if (TYPE_PRECISION (actual) == 32) | |
1259 | return uns_p ? TYPE_SUFFIX_u32 : TYPE_SUFFIX_s32; | |
1260 | if (TYPE_PRECISION (actual) == 64) | |
1261 | return uns_p ? TYPE_SUFFIX_u64 : TYPE_SUFFIX_s64; | |
1262 | } | |
1263 | ||
1264 | error_at (location, "passing %qT to argument %d of %qE, which expects" | |
1265 | " a 32-bit or 64-bit integer type", actual, argno + 1, fndecl); | |
1266 | return NUM_TYPE_SUFFIXES; | |
1267 | } | |
1268 | ||
1269 | /* Require argument ARGNO to be a pointer to a scalar type that has a | |
1270 | corresponding type suffix. Return that type suffix on success, | |
1271 | otherwise report an error and return NUM_TYPE_SUFFIXES. | |
1272 | GATHER_SCATTER_P is true if the function is a gather/scatter | |
1273 | operation, and so requires a pointer to 32-bit or 64-bit data. */ | |
1274 | type_suffix_index | |
1275 | function_resolver::infer_pointer_type (unsigned int argno, | |
1276 | bool gather_scatter_p) | |
1277 | { | |
1278 | tree actual = get_argument_type (argno); | |
1279 | if (actual == error_mark_node) | |
1280 | return NUM_TYPE_SUFFIXES; | |
1281 | ||
1282 | if (TREE_CODE (actual) != POINTER_TYPE) | |
1283 | { | |
1284 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1285 | " expects a pointer type", actual, argno + 1, fndecl); | |
1286 | if (VECTOR_TYPE_P (actual) && gather_scatter_p) | |
1287 | inform (location, "an explicit type suffix is needed" | |
1288 | " when using a vector of base addresses"); | |
1289 | return NUM_TYPE_SUFFIXES; | |
1290 | } | |
1291 | ||
1292 | tree target = TREE_TYPE (actual); | |
1293 | type_suffix_index type = find_type_suffix_for_scalar_type (target); | |
1294 | if (type == NUM_TYPE_SUFFIXES) | |
1295 | { | |
1296 | error_at (location, "passing %qT to argument %d of %qE, but %qT is not" | |
1297 | " a valid SVE element type", actual, argno + 1, fndecl, | |
0a09a948 | 1298 | build_qualified_type (target, 0)); |
624d0f07 RS |
1299 | return NUM_TYPE_SUFFIXES; |
1300 | } | |
1301 | unsigned int bits = type_suffixes[type].element_bits; | |
1302 | if (gather_scatter_p && bits != 32 && bits != 64) | |
1303 | { | |
1304 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1305 | " expects a pointer to 32-bit or 64-bit elements", | |
1306 | actual, argno + 1, fndecl); | |
1307 | return NUM_TYPE_SUFFIXES; | |
1308 | } | |
1309 | ||
1310 | return type; | |
1311 | } | |
1312 | ||
1313 | /* Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS | |
1314 | vectors; NUM_VECTORS is 1 for the former. Return the associated type | |
1315 | suffix on success, using TYPE_SUFFIX_b for predicates. Report an error | |
1316 | and return NUM_TYPE_SUFFIXES on failure. */ | |
1317 | type_suffix_index | |
1318 | function_resolver::infer_vector_or_tuple_type (unsigned int argno, | |
1319 | unsigned int num_vectors) | |
1320 | { | |
1321 | tree actual = get_argument_type (argno); | |
1322 | if (actual == error_mark_node) | |
1323 | return NUM_TYPE_SUFFIXES; | |
1324 | ||
1325 | /* A linear search should be OK here, since the code isn't hot and | |
1326 | the number of types is only small. */ | |
1327 | for (unsigned int size_i = 0; size_i < MAX_TUPLE_SIZE; ++size_i) | |
1328 | for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i) | |
1329 | { | |
1330 | vector_type_index type_i = type_suffixes[suffix_i].vector_type; | |
1331 | tree type = acle_vector_types[size_i][type_i]; | |
38e62001 | 1332 | if (type && matches_type_p (type, actual)) |
624d0f07 RS |
1333 | { |
1334 | if (size_i + 1 == num_vectors) | |
1335 | return type_suffix_index (suffix_i); | |
1336 | ||
1337 | if (num_vectors == 1) | |
1338 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1339 | " expects a single SVE vector rather than a tuple", | |
1340 | actual, argno + 1, fndecl); | |
1341 | else if (size_i == 0 && type_i != VECTOR_TYPE_svbool_t) | |
1342 | error_at (location, "passing single vector %qT to argument %d" | |
1343 | " of %qE, which expects a tuple of %d vectors", | |
1344 | actual, argno + 1, fndecl, num_vectors); | |
1345 | else | |
1346 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1347 | " expects a tuple of %d vectors", actual, argno + 1, | |
1348 | fndecl, num_vectors); | |
1349 | return NUM_TYPE_SUFFIXES; | |
1350 | } | |
1351 | } | |
1352 | ||
1353 | if (num_vectors == 1) | |
1354 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1355 | " expects an SVE vector type", actual, argno + 1, fndecl); | |
1356 | else | |
1357 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1358 | " expects an SVE tuple type", actual, argno + 1, fndecl); | |
1359 | return NUM_TYPE_SUFFIXES; | |
1360 | } | |
1361 | ||
1362 | /* Require argument ARGNO to have some form of vector type. Return the | |
1363 | associated type suffix on success, using TYPE_SUFFIX_b for predicates. | |
1364 | Report an error and return NUM_TYPE_SUFFIXES on failure. */ | |
1365 | type_suffix_index | |
1366 | function_resolver::infer_vector_type (unsigned int argno) | |
1367 | { | |
1368 | return infer_vector_or_tuple_type (argno, 1); | |
1369 | } | |
1370 | ||
1371 | /* Like infer_vector_type, but also require the type to be integral. */ | |
1372 | type_suffix_index | |
1373 | function_resolver::infer_integer_vector_type (unsigned int argno) | |
1374 | { | |
1375 | type_suffix_index type = infer_vector_type (argno); | |
1376 | if (type == NUM_TYPE_SUFFIXES) | |
1377 | return type; | |
1378 | ||
1379 | if (!type_suffixes[type].integer_p) | |
1380 | { | |
1381 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1382 | " expects a vector of integers", get_argument_type (argno), | |
1383 | argno + 1, fndecl); | |
1384 | return NUM_TYPE_SUFFIXES; | |
1385 | } | |
1386 | ||
1387 | return type; | |
1388 | } | |
1389 | ||
1390 | /* Like infer_vector_type, but also require the type to be an unsigned | |
1391 | integer. */ | |
1392 | type_suffix_index | |
1393 | function_resolver::infer_unsigned_vector_type (unsigned int argno) | |
1394 | { | |
1395 | type_suffix_index type = infer_vector_type (argno); | |
1396 | if (type == NUM_TYPE_SUFFIXES) | |
1397 | return type; | |
1398 | ||
1399 | if (!type_suffixes[type].unsigned_p) | |
1400 | { | |
1401 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1402 | " expects a vector of unsigned integers", | |
1403 | get_argument_type (argno), argno + 1, fndecl); | |
1404 | return NUM_TYPE_SUFFIXES; | |
1405 | } | |
1406 | ||
1407 | return type; | |
1408 | } | |
1409 | ||
1410 | /* Like infer_vector_type, but also require the element size to be | |
1411 | 32 or 64 bits. */ | |
1412 | type_suffix_index | |
1413 | function_resolver::infer_sd_vector_type (unsigned int argno) | |
1414 | { | |
1415 | type_suffix_index type = infer_vector_type (argno); | |
1416 | if (type == NUM_TYPE_SUFFIXES) | |
1417 | return type; | |
1418 | ||
1419 | unsigned int bits = type_suffixes[type].element_bits; | |
1420 | if (bits != 32 && bits != 64) | |
1421 | { | |
1422 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1423 | " expects a vector of 32-bit or 64-bit elements", | |
1424 | get_argument_type (argno), argno + 1, fndecl); | |
1425 | return NUM_TYPE_SUFFIXES; | |
1426 | } | |
1427 | ||
1428 | return type; | |
1429 | } | |
1430 | ||
1431 | /* If the function operates on tuples of vectors, require argument ARGNO to be | |
1432 | a tuple with the appropriate number of vectors, otherwise require it to be | |
1433 | a single vector. Return the associated type suffix on success, using | |
1434 | TYPE_SUFFIX_b for predicates. Report an error and return NUM_TYPE_SUFFIXES | |
1435 | on failure. */ | |
1436 | type_suffix_index | |
1437 | function_resolver::infer_tuple_type (unsigned int argno) | |
1438 | { | |
1439 | return infer_vector_or_tuple_type (argno, vectors_per_tuple ()); | |
1440 | } | |
1441 | ||
1442 | /* Require argument ARGNO to be a vector or scalar argument. Return true | |
1443 | if it is, otherwise report an appropriate error. */ | |
1444 | bool | |
1445 | function_resolver::require_vector_or_scalar_type (unsigned int argno) | |
1446 | { | |
1447 | tree actual = get_argument_type (argno); | |
1448 | if (actual == error_mark_node) | |
1449 | return false; | |
1450 | ||
1451 | if (!scalar_argument_p (argno) && !VECTOR_TYPE_P (actual)) | |
1452 | { | |
1453 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1454 | " expects a vector or scalar type", actual, argno + 1, fndecl); | |
1455 | return false; | |
1456 | } | |
1457 | ||
1458 | return true; | |
1459 | } | |
1460 | ||
1461 | /* Require argument ARGNO to have vector type TYPE, in cases where this | |
1462 | requirement holds for all uses of the function. Return true if the | |
1463 | argument has the right form, otherwise report an appropriate error. */ | |
1464 | bool | |
1465 | function_resolver::require_vector_type (unsigned int argno, | |
1466 | vector_type_index type) | |
1467 | { | |
1468 | tree expected = acle_vector_types[0][type]; | |
1469 | tree actual = get_argument_type (argno); | |
38e62001 | 1470 | if (!matches_type_p (expected, actual)) |
624d0f07 RS |
1471 | { |
1472 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1473 | " expects %qT", actual, argno + 1, fndecl, expected); | |
1474 | return false; | |
1475 | } | |
1476 | return true; | |
1477 | } | |
1478 | ||
1479 | /* Like require_vector_type, but TYPE is inferred from previous arguments | |
1480 | rather than being a fixed part of the function signature. This changes | |
1481 | the nature of the error messages. */ | |
1482 | bool | |
1483 | function_resolver::require_matching_vector_type (unsigned int argno, | |
1484 | type_suffix_index type) | |
1485 | { | |
1486 | type_suffix_index new_type = infer_vector_type (argno); | |
1487 | if (new_type == NUM_TYPE_SUFFIXES) | |
1488 | return false; | |
1489 | ||
1490 | if (type != new_type) | |
1491 | { | |
1492 | error_at (location, "passing %qT to argument %d of %qE, but" | |
1493 | " previous arguments had type %qT", | |
1494 | get_vector_type (new_type), argno + 1, fndecl, | |
1495 | get_vector_type (type)); | |
1496 | return false; | |
1497 | } | |
1498 | return true; | |
1499 | } | |
1500 | ||
1501 | /* Require argument ARGNO to be a vector type with the following properties: | |
1502 | ||
1503 | - the type class must be the same as FIRST_TYPE's if EXPECTED_TCLASS | |
1504 | is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself. | |
1505 | ||
1506 | - the element size must be: | |
1507 | ||
1508 | - the same as FIRST_TYPE's if EXPECTED_BITS == SAME_SIZE | |
1509 | - half of FIRST_TYPE's if EXPECTED_BITS == HALF_SIZE | |
1510 | - a quarter of FIRST_TYPE's if EXPECTED_BITS == QUARTER_SIZE | |
1511 | - EXPECTED_BITS itself otherwise | |
1512 | ||
1513 | Return true if the argument has the required type, otherwise report | |
1514 | an appropriate error. | |
1515 | ||
1516 | FIRST_ARGNO is the first argument that is known to have type FIRST_TYPE. | |
1517 | Usually it comes before ARGNO, but sometimes it is more natural to resolve | |
1518 | arguments out of order. | |
1519 | ||
1520 | If the required properties depend on FIRST_TYPE then both FIRST_ARGNO and | |
1521 | ARGNO contribute to the resolution process. If the required properties | |
1522 | are fixed, only FIRST_ARGNO contributes to the resolution process. | |
1523 | ||
1524 | This function is a bit of a Swiss army knife. The complication comes | |
1525 | from trying to give good error messages when FIRST_ARGNO and ARGNO are | |
1526 | inconsistent, since either of them might be wrong. */ | |
1527 | bool function_resolver:: | |
1528 | require_derived_vector_type (unsigned int argno, | |
1529 | unsigned int first_argno, | |
1530 | type_suffix_index first_type, | |
1531 | type_class_index expected_tclass, | |
1532 | unsigned int expected_bits) | |
1533 | { | |
1534 | /* If the type needs to match FIRST_ARGNO exactly, use the preferred | |
1535 | error message for that case. The VECTOR_TYPE_P test excludes tuple | |
1536 | types, which we handle below instead. */ | |
1537 | bool both_vectors_p = VECTOR_TYPE_P (get_argument_type (first_argno)); | |
1538 | if (both_vectors_p | |
1539 | && expected_tclass == SAME_TYPE_CLASS | |
1540 | && expected_bits == SAME_SIZE) | |
1541 | { | |
1542 | /* There's no need to resolve this case out of order. */ | |
1543 | gcc_assert (argno > first_argno); | |
1544 | return require_matching_vector_type (argno, first_type); | |
1545 | } | |
1546 | ||
1547 | /* Use FIRST_TYPE to get the expected type class and element size. */ | |
1548 | type_class_index orig_expected_tclass = expected_tclass; | |
1549 | if (expected_tclass == NUM_TYPE_CLASSES) | |
1550 | expected_tclass = type_suffixes[first_type].tclass; | |
1551 | ||
1552 | unsigned int orig_expected_bits = expected_bits; | |
1553 | if (expected_bits == SAME_SIZE) | |
1554 | expected_bits = type_suffixes[first_type].element_bits; | |
1555 | else if (expected_bits == HALF_SIZE) | |
1556 | expected_bits = type_suffixes[first_type].element_bits / 2; | |
1557 | else if (expected_bits == QUARTER_SIZE) | |
1558 | expected_bits = type_suffixes[first_type].element_bits / 4; | |
1559 | ||
1560 | /* If the expected type doesn't depend on FIRST_TYPE at all, | |
1561 | just check for the fixed choice of vector type. */ | |
1562 | if (expected_tclass == orig_expected_tclass | |
1563 | && expected_bits == orig_expected_bits) | |
1564 | { | |
1565 | const type_suffix_info &expected_suffix | |
1566 | = type_suffixes[find_type_suffix (expected_tclass, expected_bits)]; | |
1567 | return require_vector_type (argno, expected_suffix.vector_type); | |
1568 | } | |
1569 | ||
1570 | /* Require the argument to be some form of SVE vector type, | |
1571 | without being specific about the type of vector we want. */ | |
1572 | type_suffix_index actual_type = infer_vector_type (argno); | |
1573 | if (actual_type == NUM_TYPE_SUFFIXES) | |
1574 | return false; | |
1575 | ||
1576 | /* Exit now if we got the right type. */ | |
1577 | bool tclass_ok_p = (type_suffixes[actual_type].tclass == expected_tclass); | |
1578 | bool size_ok_p = (type_suffixes[actual_type].element_bits == expected_bits); | |
1579 | if (tclass_ok_p && size_ok_p) | |
1580 | return true; | |
1581 | ||
1582 | /* First look for cases in which the actual type contravenes a fixed | |
1583 | size requirement, without having to refer to FIRST_TYPE. */ | |
1584 | if (!size_ok_p && expected_bits == orig_expected_bits) | |
1585 | { | |
1586 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1587 | " expects a vector of %d-bit elements", | |
1588 | get_vector_type (actual_type), argno + 1, fndecl, | |
1589 | expected_bits); | |
1590 | return false; | |
1591 | } | |
1592 | ||
1593 | /* Likewise for a fixed type class requirement. This is only ever | |
1594 | needed for signed and unsigned types, so don't create unnecessary | |
1595 | translation work for other type classes. */ | |
1596 | if (!tclass_ok_p && orig_expected_tclass == TYPE_signed) | |
1597 | { | |
1598 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1599 | " expects a vector of signed integers", | |
1600 | get_vector_type (actual_type), argno + 1, fndecl); | |
1601 | return false; | |
1602 | } | |
1603 | if (!tclass_ok_p && orig_expected_tclass == TYPE_unsigned) | |
1604 | { | |
1605 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1606 | " expects a vector of unsigned integers", | |
1607 | get_vector_type (actual_type), argno + 1, fndecl); | |
1608 | return false; | |
1609 | } | |
1610 | ||
1611 | /* Make sure that FIRST_TYPE itself is sensible before using it | |
1612 | as a basis for an error message. */ | |
1613 | if (resolve_to (mode_suffix_id, first_type) == error_mark_node) | |
1614 | return false; | |
1615 | ||
1616 | /* If the arguments have consistent type classes, but a link between | |
1617 | the sizes has been broken, try to describe the error in those terms. */ | |
1618 | if (both_vectors_p && tclass_ok_p && orig_expected_bits == SAME_SIZE) | |
1619 | { | |
1620 | if (argno < first_argno) | |
1621 | { | |
1622 | std::swap (argno, first_argno); | |
1623 | std::swap (actual_type, first_type); | |
1624 | } | |
1625 | error_at (location, "arguments %d and %d of %qE must have the" | |
1626 | " same element size, but the values passed here have type" | |
1627 | " %qT and %qT respectively", first_argno + 1, argno + 1, | |
1628 | fndecl, get_vector_type (first_type), | |
1629 | get_vector_type (actual_type)); | |
1630 | return false; | |
1631 | } | |
1632 | ||
1633 | /* Likewise in reverse: look for cases in which the sizes are consistent | |
1634 | but a link between the type classes has been broken. */ | |
1635 | if (both_vectors_p | |
1636 | && size_ok_p | |
1637 | && orig_expected_tclass == SAME_TYPE_CLASS | |
1638 | && type_suffixes[first_type].integer_p | |
1639 | && type_suffixes[actual_type].integer_p) | |
1640 | { | |
1641 | if (argno < first_argno) | |
1642 | { | |
1643 | std::swap (argno, first_argno); | |
1644 | std::swap (actual_type, first_type); | |
1645 | } | |
1646 | error_at (location, "arguments %d and %d of %qE must have the" | |
1647 | " same signedness, but the values passed here have type" | |
1648 | " %qT and %qT respectively", first_argno + 1, argno + 1, | |
1649 | fndecl, get_vector_type (first_type), | |
1650 | get_vector_type (actual_type)); | |
1651 | return false; | |
1652 | } | |
1653 | ||
1654 | /* The two arguments are wildly inconsistent. */ | |
1655 | type_suffix_index expected_type | |
1656 | = find_type_suffix (expected_tclass, expected_bits); | |
1657 | error_at (location, "passing %qT instead of the expected %qT to argument" | |
1658 | " %d of %qE, after passing %qT to argument %d", | |
1659 | get_vector_type (actual_type), get_vector_type (expected_type), | |
1660 | argno + 1, fndecl, get_argument_type (first_argno), | |
1661 | first_argno + 1); | |
1662 | return false; | |
1663 | } | |
1664 | ||
0a09a948 RS |
1665 | /* Require argument ARGNO to match argument FIRST_ARGNO, which was inferred |
1666 | to be a pointer to a scalar element of type TYPE. */ | |
1667 | bool | |
1668 | function_resolver::require_matching_pointer_type (unsigned int argno, | |
1669 | unsigned int first_argno, | |
1670 | type_suffix_index type) | |
1671 | { | |
1672 | type_suffix_index new_type = infer_pointer_type (argno); | |
1673 | if (new_type == NUM_TYPE_SUFFIXES) | |
1674 | return false; | |
1675 | ||
1676 | if (type != new_type) | |
1677 | { | |
1678 | error_at (location, "passing %qT to argument %d of %qE, but" | |
1679 | " argument %d had type %qT", get_argument_type (argno), | |
1680 | argno + 1, fndecl, first_argno + 1, | |
1681 | get_argument_type (first_argno)); | |
1682 | return false; | |
1683 | } | |
1684 | return true; | |
1685 | } | |
1686 | ||
624d0f07 RS |
1687 | /* Require argument ARGNO to be a (possibly variable) scalar, using EXPECTED |
1688 | as the name of its expected type. Return true if the argument has the | |
1689 | right form, otherwise report an appropriate error. */ | |
1690 | bool | |
1691 | function_resolver::require_scalar_type (unsigned int argno, | |
1692 | const char *expected) | |
1693 | { | |
1694 | if (!scalar_argument_p (argno)) | |
1695 | { | |
1696 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1697 | " expects %qs", get_argument_type (argno), argno + 1, | |
1698 | fndecl, expected); | |
1699 | return false; | |
1700 | } | |
1701 | return true; | |
1702 | } | |
1703 | ||
1704 | /* Require argument ARGNO to be some form of pointer, without being specific | |
1705 | about its target type. Return true if the argument has the right form, | |
1706 | otherwise report an appropriate error. */ | |
1707 | bool | |
1708 | function_resolver::require_pointer_type (unsigned int argno) | |
1709 | { | |
1710 | if (!scalar_argument_p (argno)) | |
1711 | { | |
1712 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1713 | " expects a scalar pointer", get_argument_type (argno), | |
1714 | argno + 1, fndecl); | |
1715 | return false; | |
1716 | } | |
1717 | return true; | |
1718 | } | |
1719 | ||
1720 | /* Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE, and argument | |
1721 | ARGNO should be consistent with it. Return true if it is, otherwise | |
1722 | report an appropriate error. */ | |
1723 | bool function_resolver:: | |
1724 | require_matching_integer_scalar_type (unsigned int argno, | |
1725 | unsigned int first_argno, | |
1726 | type_suffix_index expected_type) | |
1727 | { | |
1728 | type_suffix_index actual_type = infer_integer_scalar_type (argno); | |
1729 | if (actual_type == NUM_TYPE_SUFFIXES) | |
1730 | return false; | |
1731 | ||
1732 | if (actual_type == expected_type) | |
1733 | return true; | |
1734 | ||
1735 | error_at (location, "call to %qE is ambiguous; argument %d has type" | |
1736 | " %qs but argument %d has type %qs", fndecl, | |
1737 | first_argno + 1, get_scalar_type_name (expected_type), | |
1738 | argno + 1, get_scalar_type_name (actual_type)); | |
1739 | return false; | |
1740 | } | |
1741 | ||
1742 | /* Require argument ARGNO to be a (possibly variable) scalar, expecting it | |
1743 | to have the following properties: | |
1744 | ||
1745 | - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS | |
1746 | is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself. | |
1747 | ||
1748 | - the element size must be the same as for type suffix 0 if EXPECTED_BITS | |
1749 | is SAME_TYPE_SIZE, otherwise it must be EXPECTED_BITS itself. | |
1750 | ||
1751 | Return true if the argument is valid, otherwise report an appropriate error. | |
1752 | ||
1753 | Note that we don't check whether the scalar type actually has the required | |
1754 | properties, since that's subject to implicit promotions and conversions. | |
1755 | Instead we just use the expected properties to tune the error message. */ | |
1756 | bool function_resolver:: | |
1757 | require_derived_scalar_type (unsigned int argno, | |
1758 | type_class_index expected_tclass, | |
1759 | unsigned int expected_bits) | |
1760 | { | |
1761 | gcc_assert (expected_tclass == SAME_TYPE_CLASS | |
1762 | || expected_tclass == TYPE_signed | |
1763 | || expected_tclass == TYPE_unsigned); | |
1764 | ||
1765 | /* If the expected type doesn't depend on the type suffix at all, | |
1766 | just check for the fixed choice of scalar type. */ | |
1767 | if (expected_tclass != SAME_TYPE_CLASS && expected_bits != SAME_SIZE) | |
1768 | { | |
1769 | type_suffix_index expected_type | |
1770 | = find_type_suffix (expected_tclass, expected_bits); | |
1771 | return require_scalar_type (argno, get_scalar_type_name (expected_type)); | |
1772 | } | |
1773 | ||
1774 | if (scalar_argument_p (argno)) | |
1775 | return true; | |
1776 | ||
1777 | if (expected_tclass == SAME_TYPE_CLASS) | |
1778 | /* It doesn't really matter whether the element is expected to be | |
1779 | the same size as type suffix 0. */ | |
1780 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1781 | " expects a scalar element", get_argument_type (argno), | |
1782 | argno + 1, fndecl); | |
1783 | else | |
1784 | /* It doesn't seem useful to distinguish between signed and unsigned | |
1785 | scalars here. */ | |
1786 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1787 | " expects a scalar integer", get_argument_type (argno), | |
1788 | argno + 1, fndecl); | |
1789 | return false; | |
1790 | } | |
1791 | ||
1792 | /* Require argument ARGNO to be suitable for an integer constant expression. | |
1793 | Return true if it is, otherwise report an appropriate error. | |
1794 | ||
1795 | function_checker checks whether the argument is actually constant and | |
1796 | has a suitable range. The reason for distinguishing immediate arguments | |
1797 | here is because it provides more consistent error messages than | |
1798 | require_scalar_type would. */ | |
1799 | bool | |
1800 | function_resolver::require_integer_immediate (unsigned int argno) | |
1801 | { | |
1802 | if (!scalar_argument_p (argno)) | |
1803 | { | |
1804 | report_non_ice (location, fndecl, argno); | |
1805 | return false; | |
1806 | } | |
1807 | return true; | |
1808 | } | |
1809 | ||
1810 | /* Require argument ARGNO to be a vector base in a gather-style address. | |
1811 | Return its type on success, otherwise return NUM_VECTOR_TYPES. */ | |
1812 | vector_type_index | |
1813 | function_resolver::infer_vector_base_type (unsigned int argno) | |
1814 | { | |
1815 | type_suffix_index type = infer_vector_type (argno); | |
1816 | if (type == NUM_TYPE_SUFFIXES) | |
1817 | return NUM_VECTOR_TYPES; | |
1818 | ||
1819 | if (type == TYPE_SUFFIX_u32 || type == TYPE_SUFFIX_u64) | |
1820 | return type_suffixes[type].vector_type; | |
1821 | ||
1822 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1823 | " expects %qs or %qs", get_argument_type (argno), | |
1824 | argno + 1, fndecl, "svuint32_t", "svuint64_t"); | |
1825 | return NUM_VECTOR_TYPES; | |
1826 | } | |
1827 | ||
1828 | /* Require argument ARGNO to be a vector displacement in a gather-style | |
1829 | address. Return its type on success, otherwise return NUM_VECTOR_TYPES. */ | |
1830 | vector_type_index | |
1831 | function_resolver::infer_vector_displacement_type (unsigned int argno) | |
1832 | { | |
1833 | type_suffix_index type = infer_integer_vector_type (argno); | |
1834 | if (type == NUM_TYPE_SUFFIXES) | |
1835 | return NUM_VECTOR_TYPES; | |
1836 | ||
1837 | if (type_suffixes[type].integer_p | |
1838 | && (type_suffixes[type].element_bits == 32 | |
1839 | || type_suffixes[type].element_bits == 64)) | |
1840 | return type_suffixes[type].vector_type; | |
1841 | ||
1842 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1843 | " expects a vector of 32-bit or 64-bit integers", | |
1844 | get_argument_type (argno), argno + 1, fndecl); | |
1845 | return NUM_VECTOR_TYPES; | |
1846 | } | |
1847 | ||
1848 | /* Require argument ARGNO to be a vector displacement in a gather-style | |
1849 | address. There are three possible uses: | |
1850 | ||
1851 | - for loading into elements of type TYPE (when LOAD_P is true) | |
1852 | - for storing from elements of type TYPE (when LOAD_P is false) | |
1853 | - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES) | |
1854 | ||
1855 | The overloaded function's mode suffix determines the units of the | |
1856 | displacement (bytes for "_offset", elements for "_index"). | |
1857 | ||
1858 | Return the associated mode on success, otherwise report an error | |
1859 | and return MODE_none. */ | |
1860 | mode_suffix_index | |
1861 | function_resolver::resolve_sv_displacement (unsigned int argno, | |
1862 | type_suffix_index type, | |
1863 | bool load_p) | |
1864 | { | |
1865 | if (type == NUM_TYPE_SUFFIXES) | |
1866 | { | |
1867 | /* For prefetches, the base is a void pointer and the displacement | |
1868 | can be any valid offset or index type. */ | |
1869 | vector_type_index displacement_vector_type | |
1870 | = infer_vector_displacement_type (argno); | |
1871 | if (displacement_vector_type == NUM_VECTOR_TYPES) | |
1872 | return MODE_none; | |
1873 | ||
1874 | mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES, | |
1875 | displacement_vector_type, | |
1876 | displacement_units ()); | |
1877 | gcc_assert (mode != MODE_none); | |
1878 | return mode; | |
1879 | } | |
1880 | ||
0a09a948 RS |
1881 | unsigned int required_bits = type_suffixes[type].element_bits; |
1882 | if (required_bits == 32 | |
1883 | && displacement_units () == UNITS_elements | |
1884 | && !lookup_form (MODE_s32index, type) | |
1885 | && !lookup_form (MODE_u32index, type)) | |
1886 | { | |
1887 | if (lookup_form (MODE_u32base_index, type)) | |
1888 | { | |
1889 | if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) | |
1890 | { | |
1891 | gcc_assert (!load_p); | |
1892 | error_at (location, "when storing %qT, %qE requires a vector" | |
1893 | " base and a scalar index", get_vector_type (type), | |
1894 | fndecl); | |
1895 | } | |
1896 | else | |
1897 | error_at (location, "%qE requires a vector base and a scalar" | |
1898 | " index", fndecl); | |
1899 | } | |
1900 | else | |
1901 | error_at (location, "%qE does not support 32-bit vector type %qT", | |
1902 | fndecl, get_vector_type (type)); | |
1903 | return MODE_none; | |
1904 | } | |
1905 | ||
624d0f07 RS |
1906 | /* Check for some form of vector type, without naming any in particular |
1907 | as being expected. */ | |
1908 | type_suffix_index displacement_type = infer_vector_type (argno); | |
1909 | if (displacement_type == NUM_TYPE_SUFFIXES) | |
1910 | return MODE_none; | |
1911 | ||
1912 | /* If the displacement type is consistent with the data vector type, | |
1913 | try to find the associated mode suffix. This will fall through | |
1914 | for non-integral displacement types. */ | |
624d0f07 RS |
1915 | if (type_suffixes[displacement_type].element_bits == required_bits) |
1916 | { | |
1917 | vector_type_index displacement_vector_type | |
1918 | = type_suffixes[displacement_type].vector_type; | |
1919 | mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES, | |
1920 | displacement_vector_type, | |
1921 | displacement_units ()); | |
1922 | if (mode != MODE_none) | |
0a09a948 RS |
1923 | { |
1924 | if (mode == MODE_s32offset | |
1925 | && !lookup_form (mode, type) | |
1926 | && lookup_form (MODE_u32offset, type)) | |
1927 | { | |
1928 | if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) | |
1929 | error_at (location, "%qE does not support 32-bit sign-extended" | |
1930 | " offsets", fndecl); | |
1931 | else | |
1932 | error_at (location, "%qE does not support sign-extended" | |
1933 | " offsets", fndecl); | |
1934 | return MODE_none; | |
1935 | } | |
1936 | return mode; | |
1937 | } | |
624d0f07 RS |
1938 | } |
1939 | ||
1940 | if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) | |
1941 | { | |
1942 | /* TYPE has been inferred rather than specified by the user, | |
1943 | so mention it in the error messages. */ | |
1944 | if (load_p) | |
1945 | error_at (location, "passing %qT to argument %d of %qE, which when" | |
1946 | " loading %qT expects a vector of %d-bit integers", | |
1947 | get_argument_type (argno), argno + 1, fndecl, | |
1948 | get_vector_type (type), required_bits); | |
1949 | else | |
1950 | error_at (location, "passing %qT to argument %d of %qE, which when" | |
1951 | " storing %qT expects a vector of %d-bit integers", | |
1952 | get_argument_type (argno), argno + 1, fndecl, | |
1953 | get_vector_type (type), required_bits); | |
1954 | } | |
1955 | else | |
1956 | /* TYPE is part of the function name. */ | |
1957 | error_at (location, "passing %qT to argument %d of %qE, which" | |
1958 | " expects a vector of %d-bit integers", | |
1959 | get_argument_type (argno), argno + 1, fndecl, required_bits); | |
1960 | return MODE_none; | |
1961 | } | |
1962 | ||
1963 | /* Require the arguments starting at ARGNO to form a gather-style address. | |
1964 | There are three possible uses: | |
1965 | ||
1966 | - for loading into elements of type TYPE (when LOAD_P is true) | |
1967 | - for storing from elements of type TYPE (when LOAD_P is false) | |
1968 | - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES) | |
1969 | ||
1970 | The three possible addresses are: | |
1971 | ||
1972 | - a vector base with no displacement | |
1973 | - a vector base and a scalar displacement | |
1974 | - a scalar (pointer) base and a vector displacement | |
1975 | ||
1976 | The overloaded function's mode suffix determines whether there is | |
1977 | a displacement, and if so, what units it uses: | |
1978 | ||
1979 | - MODE_none: no displacement | |
1980 | - MODE_offset: the displacement is measured in bytes | |
1981 | - MODE_index: the displacement is measured in elements | |
1982 | ||
1983 | Return the mode of the non-overloaded function on success, otherwise | |
1984 | report an error and return MODE_none. */ | |
1985 | mode_suffix_index | |
1986 | function_resolver::resolve_gather_address (unsigned int argno, | |
1987 | type_suffix_index type, | |
1988 | bool load_p) | |
1989 | { | |
1990 | tree actual = get_argument_type (argno); | |
1991 | if (actual == error_mark_node) | |
1992 | return MODE_none; | |
1993 | ||
1994 | if (displacement_units () != UNITS_none) | |
1995 | { | |
1996 | /* Some form of displacement is needed. First handle a scalar | |
1997 | pointer base and a vector displacement. */ | |
1998 | if (scalar_argument_p (argno)) | |
1999 | /* Don't check the pointer type here, since there's only one valid | |
2000 | choice. Leave that to the frontend. */ | |
2001 | return resolve_sv_displacement (argno + 1, type, load_p); | |
2002 | ||
2003 | if (!VECTOR_TYPE_P (actual)) | |
2004 | { | |
2005 | error_at (location, "passing %qT to argument %d of %qE," | |
2006 | " which expects a vector or pointer base address", | |
2007 | actual, argno + 1, fndecl); | |
2008 | return MODE_none; | |
2009 | } | |
2010 | } | |
2011 | ||
2012 | /* Check for the correct choice of vector base type. */ | |
2013 | vector_type_index base_vector_type; | |
2014 | if (type == NUM_TYPE_SUFFIXES) | |
2015 | { | |
2016 | /* Since prefetches have no type suffix, there is a free choice | |
2017 | between 32-bit and 64-bit base addresses. */ | |
2018 | base_vector_type = infer_vector_base_type (argno); | |
2019 | if (base_vector_type == NUM_VECTOR_TYPES) | |
2020 | return MODE_none; | |
2021 | } | |
2022 | else | |
2023 | { | |
2024 | /* Check for some form of vector type, without saying which type | |
2025 | we expect. */ | |
2026 | type_suffix_index base_type = infer_vector_type (argno); | |
2027 | if (base_type == NUM_TYPE_SUFFIXES) | |
2028 | return MODE_none; | |
2029 | ||
2030 | /* Check whether the type is the right one. */ | |
2031 | unsigned int required_bits = type_suffixes[type].element_bits; | |
2032 | gcc_assert (required_bits == 32 || required_bits == 64); | |
2033 | type_suffix_index required_type = (required_bits == 32 | |
2034 | ? TYPE_SUFFIX_u32 | |
2035 | : TYPE_SUFFIX_u64); | |
2036 | if (required_type != base_type) | |
2037 | { | |
2038 | error_at (location, "passing %qT to argument %d of %qE," | |
2039 | " which expects %qT", actual, argno + 1, fndecl, | |
2040 | get_vector_type (required_type)); | |
2041 | return MODE_none; | |
2042 | } | |
2043 | base_vector_type = type_suffixes[base_type].vector_type; | |
2044 | } | |
2045 | ||
2046 | /* Check the scalar displacement, if any. */ | |
2047 | if (displacement_units () != UNITS_none | |
2048 | && !require_scalar_type (argno + 1, "int64_t")) | |
2049 | return MODE_none; | |
2050 | ||
2051 | /* Find the appropriate mode suffix. The checks above should have | |
2052 | weeded out all erroneous cases. */ | |
2053 | for (unsigned int mode_i = 0; mode_i < ARRAY_SIZE (mode_suffixes); ++mode_i) | |
2054 | { | |
2055 | const mode_suffix_info &mode = mode_suffixes[mode_i]; | |
2056 | if (mode.base_vector_type == base_vector_type | |
2057 | && mode.displacement_vector_type == NUM_VECTOR_TYPES | |
2058 | && mode.displacement_units == displacement_units ()) | |
2059 | return mode_suffix_index (mode_i); | |
2060 | } | |
2061 | ||
2062 | gcc_unreachable (); | |
2063 | } | |
2064 | ||
2065 | /* Require arguments ARGNO and ARGNO + 1 to form an ADR-style address, | |
2066 | i.e. one with a vector of base addresses and a vector of displacements. | |
2067 | The overloaded function's mode suffix determines the units of the | |
2068 | displacement (bytes for "_offset", elements for "_index"). | |
2069 | ||
2070 | Return the associated mode suffix on success, otherwise report | |
2071 | an error and return MODE_none. */ | |
2072 | mode_suffix_index | |
2073 | function_resolver::resolve_adr_address (unsigned int argno) | |
2074 | { | |
2075 | vector_type_index base_type = infer_vector_base_type (argno); | |
2076 | if (base_type == NUM_VECTOR_TYPES) | |
2077 | return MODE_none; | |
2078 | ||
2079 | vector_type_index displacement_type | |
2080 | = infer_vector_displacement_type (argno + 1); | |
2081 | if (displacement_type == NUM_VECTOR_TYPES) | |
2082 | return MODE_none; | |
2083 | ||
2084 | mode_suffix_index mode = find_mode_suffix (base_type, displacement_type, | |
2085 | displacement_units ()); | |
2086 | if (mode == MODE_none) | |
2087 | { | |
2088 | if (mode_suffix_id == MODE_offset) | |
2089 | error_at (location, "cannot combine a base of type %qT with" | |
2090 | " an offset of type %qT", | |
2091 | get_argument_type (argno), get_argument_type (argno + 1)); | |
2092 | else | |
2093 | error_at (location, "cannot combine a base of type %qT with" | |
2094 | " an index of type %qT", | |
2095 | get_argument_type (argno), get_argument_type (argno + 1)); | |
2096 | } | |
2097 | return mode; | |
2098 | } | |
2099 | ||
2100 | /* Require the function to have exactly EXPECTED arguments. Return true | |
2101 | if it does, otherwise report an appropriate error. */ | |
2102 | bool | |
2103 | function_resolver::check_num_arguments (unsigned int expected) | |
2104 | { | |
2105 | if (m_arglist.length () < expected) | |
2106 | error_at (location, "too few arguments to function %qE", fndecl); | |
2107 | else if (m_arglist.length () > expected) | |
2108 | error_at (location, "too many arguments to function %qE", fndecl); | |
2109 | return m_arglist.length () == expected; | |
2110 | } | |
2111 | ||
2112 | /* If the function is predicated, check that the first argument is a | |
2113 | suitable governing predicate. Also check that there are NOPS further | |
2114 | arguments after any governing predicate, but don't check what they are. | |
2115 | ||
2116 | Return true on success, otherwise report a suitable error. | |
2117 | When returning true: | |
2118 | ||
2119 | - set I to the number of the first unchecked argument. | |
2120 | - set NARGS to the total number of arguments. */ | |
2121 | bool | |
2122 | function_resolver::check_gp_argument (unsigned int nops, | |
2123 | unsigned int &i, unsigned int &nargs) | |
2124 | { | |
2125 | i = 0; | |
2126 | if (pred != PRED_none) | |
2127 | { | |
2128 | /* Unary merge operations should use resolve_unary instead. */ | |
2129 | gcc_assert (nops != 1 || pred != PRED_m); | |
2130 | nargs = nops + 1; | |
2131 | if (!check_num_arguments (nargs) | |
2132 | || !require_vector_type (i, VECTOR_TYPE_svbool_t)) | |
2133 | return false; | |
2134 | i += 1; | |
2135 | } | |
2136 | else | |
2137 | { | |
2138 | nargs = nops; | |
2139 | if (!check_num_arguments (nargs)) | |
2140 | return false; | |
2141 | } | |
2142 | ||
2143 | return true; | |
2144 | } | |
2145 | ||
2146 | /* Finish resolving a function whose final argument can be a vector | |
2147 | or a scalar, with the function having an implicit "_n" suffix | |
2148 | in the latter case. This "_n" form might only exist for certain | |
2149 | type suffixes. | |
2150 | ||
0a09a948 RS |
2151 | ARGNO is the index of the final argument. The inferred type suffix |
2152 | was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE. | |
624d0f07 RS |
2153 | EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties |
2154 | of the final vector or scalar argument, in the same way as for | |
0a09a948 RS |
2155 | require_derived_vector_type. INFERRED_TYPE is the inferred type |
2156 | suffix itself, or NUM_TYPE_SUFFIXES if it's the same as FIRST_TYPE. | |
624d0f07 RS |
2157 | |
2158 | Return the function decl of the resolved function on success, | |
2159 | otherwise report a suitable error and return error_mark_node. */ | |
2160 | tree function_resolver:: | |
2161 | finish_opt_n_resolution (unsigned int argno, unsigned int first_argno, | |
2162 | type_suffix_index first_type, | |
2163 | type_class_index expected_tclass, | |
0a09a948 RS |
2164 | unsigned int expected_bits, |
2165 | type_suffix_index inferred_type) | |
624d0f07 | 2166 | { |
0a09a948 RS |
2167 | if (inferred_type == NUM_TYPE_SUFFIXES) |
2168 | inferred_type = first_type; | |
2169 | tree scalar_form = lookup_form (MODE_n, inferred_type); | |
624d0f07 RS |
2170 | |
2171 | /* Allow the final argument to be scalar, if an _n form exists. */ | |
2172 | if (scalar_argument_p (argno)) | |
2173 | { | |
2174 | if (scalar_form) | |
2175 | return scalar_form; | |
2176 | ||
2177 | /* Check the vector form normally. If that succeeds, raise an | |
2178 | error about having no corresponding _n form. */ | |
0a09a948 | 2179 | tree res = resolve_to (mode_suffix_id, inferred_type); |
624d0f07 RS |
2180 | if (res != error_mark_node) |
2181 | error_at (location, "passing %qT to argument %d of %qE, but its" | |
2182 | " %qT form does not accept scalars", | |
2183 | get_argument_type (argno), argno + 1, fndecl, | |
2184 | get_vector_type (first_type)); | |
2185 | return error_mark_node; | |
2186 | } | |
2187 | ||
2188 | /* If an _n form does exist, provide a more accurate message than | |
2189 | require_derived_vector_type would for arguments that are neither | |
2190 | vectors nor scalars. */ | |
2191 | if (scalar_form && !require_vector_or_scalar_type (argno)) | |
2192 | return error_mark_node; | |
2193 | ||
2194 | /* Check for the correct vector type. */ | |
2195 | if (!require_derived_vector_type (argno, first_argno, first_type, | |
2196 | expected_tclass, expected_bits)) | |
2197 | return error_mark_node; | |
2198 | ||
0a09a948 | 2199 | return resolve_to (mode_suffix_id, inferred_type); |
624d0f07 RS |
2200 | } |
2201 | ||
2202 | /* Resolve a (possibly predicated) unary function. If the function uses | |
0a09a948 RS |
2203 | merge predication or if TREAT_AS_MERGE_P is true, there is an extra |
2204 | vector argument before the governing predicate that specifies the | |
2205 | values of inactive elements. This argument has the following | |
2206 | properties: | |
624d0f07 RS |
2207 | |
2208 | - the type class must be the same as for active elements if MERGE_TCLASS | |
2209 | is SAME_TYPE_CLASS, otherwise it must be MERGE_TCLASS itself. | |
2210 | ||
2211 | - the element size must be the same as for active elements if MERGE_BITS | |
2212 | is SAME_TYPE_SIZE, otherwise it must be MERGE_BITS itself. | |
2213 | ||
2214 | Return the function decl of the resolved function on success, | |
2215 | otherwise report a suitable error and return error_mark_node. */ | |
2216 | tree | |
2217 | function_resolver::resolve_unary (type_class_index merge_tclass, | |
0a09a948 RS |
2218 | unsigned int merge_bits, |
2219 | bool treat_as_merge_p) | |
624d0f07 RS |
2220 | { |
2221 | type_suffix_index type; | |
0a09a948 | 2222 | if (pred == PRED_m || treat_as_merge_p) |
624d0f07 RS |
2223 | { |
2224 | if (!check_num_arguments (3)) | |
2225 | return error_mark_node; | |
2226 | if (merge_tclass == SAME_TYPE_CLASS && merge_bits == SAME_SIZE) | |
2227 | { | |
2228 | /* The inactive elements are the same as the active elements, | |
2229 | so we can use normal left-to-right resolution. */ | |
2230 | if ((type = infer_vector_type (0)) == NUM_TYPE_SUFFIXES | |
2231 | || !require_vector_type (1, VECTOR_TYPE_svbool_t) | |
2232 | || !require_matching_vector_type (2, type)) | |
2233 | return error_mark_node; | |
2234 | } | |
2235 | else | |
2236 | { | |
2237 | /* The inactive element type is a function of the active one, | |
2238 | so resolve the active one first. */ | |
2239 | if (!require_vector_type (1, VECTOR_TYPE_svbool_t) | |
2240 | || (type = infer_vector_type (2)) == NUM_TYPE_SUFFIXES | |
2241 | || !require_derived_vector_type (0, 2, type, merge_tclass, | |
2242 | merge_bits)) | |
2243 | return error_mark_node; | |
2244 | } | |
2245 | } | |
2246 | else | |
2247 | { | |
2248 | /* We just need to check the predicate (if any) and the single | |
2249 | vector argument. */ | |
2250 | unsigned int i, nargs; | |
2251 | if (!check_gp_argument (1, i, nargs) | |
2252 | || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) | |
2253 | return error_mark_node; | |
2254 | } | |
2255 | ||
2256 | /* Handle convert-like functions in which the first type suffix is | |
2257 | explicit. */ | |
2258 | if (type_suffix_ids[0] != NUM_TYPE_SUFFIXES) | |
2259 | return resolve_to (mode_suffix_id, type_suffix_ids[0], type); | |
2260 | ||
2261 | return resolve_to (mode_suffix_id, type); | |
2262 | } | |
2263 | ||
2264 | /* Resolve a (possibly predicated) function that takes NOPS like-typed | |
2265 | vector arguments followed by NIMM integer immediates. Return the | |
2266 | function decl of the resolved function on success, otherwise report | |
2267 | a suitable error and return error_mark_node. */ | |
2268 | tree | |
2269 | function_resolver::resolve_uniform (unsigned int nops, unsigned int nimm) | |
2270 | { | |
2271 | unsigned int i, nargs; | |
2272 | type_suffix_index type; | |
2273 | if (!check_gp_argument (nops + nimm, i, nargs) | |
2274 | || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) | |
2275 | return error_mark_node; | |
2276 | ||
2277 | i += 1; | |
2278 | for (; i < nargs - nimm; ++i) | |
2279 | if (!require_matching_vector_type (i, type)) | |
2280 | return error_mark_node; | |
2281 | ||
2282 | for (; i < nargs; ++i) | |
2283 | if (!require_integer_immediate (i)) | |
2284 | return error_mark_node; | |
2285 | ||
2286 | return resolve_to (mode_suffix_id, type); | |
2287 | } | |
2288 | ||
2289 | /* Resolve a (possibly predicated) function that offers a choice between | |
2290 | taking: | |
2291 | ||
2292 | - NOPS like-typed vector arguments or | |
2293 | - NOPS - 1 like-typed vector arguments followed by a scalar argument | |
2294 | ||
2295 | Return the function decl of the resolved function on success, | |
2296 | otherwise report a suitable error and return error_mark_node. */ | |
2297 | tree | |
2298 | function_resolver::resolve_uniform_opt_n (unsigned int nops) | |
2299 | { | |
2300 | unsigned int i, nargs; | |
2301 | type_suffix_index type; | |
2302 | if (!check_gp_argument (nops, i, nargs) | |
2303 | || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) | |
2304 | return error_mark_node; | |
2305 | ||
2306 | unsigned int first_arg = i++; | |
2307 | for (; i < nargs - 1; ++i) | |
2308 | if (!require_matching_vector_type (i, type)) | |
2309 | return error_mark_node; | |
2310 | ||
2311 | return finish_opt_n_resolution (i, first_arg, type); | |
2312 | } | |
2313 | ||
2314 | /* If the call is erroneous, report an appropriate error and return | |
2315 | error_mark_node. Otherwise, if the function is overloaded, return | |
2316 | the decl of the non-overloaded function. Return NULL_TREE otherwise, | |
2317 | indicating that the call should be processed in the normal way. */ | |
2318 | tree | |
2319 | function_resolver::resolve () | |
2320 | { | |
2321 | return shape->resolve (*this); | |
2322 | } | |
2323 | ||
2324 | function_checker::function_checker (location_t location, | |
2325 | const function_instance &instance, | |
2326 | tree fndecl, tree fntype, | |
2327 | unsigned int nargs, tree *args) | |
2328 | : function_call_info (location, instance, fndecl), | |
2329 | m_fntype (fntype), m_nargs (nargs), m_args (args), | |
2330 | /* We don't have to worry about unary _m operations here, since they | |
2331 | never have arguments that need checking. */ | |
2332 | m_base_arg (pred != PRED_none ? 1 : 0) | |
2333 | { | |
2334 | } | |
2335 | ||
2336 | /* Return true if argument ARGNO exists. which it might not for | |
2337 | erroneous calls. It is safe to wave through checks if this | |
2338 | function returns false. */ | |
2339 | bool | |
2340 | function_checker::argument_exists_p (unsigned int argno) | |
2341 | { | |
2342 | gcc_assert (argno < (unsigned int) type_num_arguments (m_fntype)); | |
2343 | return argno < m_nargs; | |
2344 | } | |
2345 | ||
2346 | /* Check that argument ARGNO is an integer constant expression and | |
2347 | store its value in VALUE_OUT if so. The caller should first | |
2348 | check that argument ARGNO exists. */ | |
2349 | bool | |
2350 | function_checker::require_immediate (unsigned int argno, | |
2351 | HOST_WIDE_INT &value_out) | |
2352 | { | |
2353 | gcc_assert (argno < m_nargs); | |
2354 | tree arg = m_args[argno]; | |
2355 | ||
2356 | /* The type and range are unsigned, so read the argument as an | |
2357 | unsigned rather than signed HWI. */ | |
2358 | if (!tree_fits_uhwi_p (arg)) | |
2359 | { | |
2360 | report_non_ice (location, fndecl, argno); | |
2361 | return false; | |
2362 | } | |
2363 | ||
2364 | /* ...but treat VALUE_OUT as signed for error reporting, since printing | |
2365 | -1 is more user-friendly than the maximum uint64_t value. */ | |
2366 | value_out = tree_to_uhwi (arg); | |
2367 | return true; | |
2368 | } | |
2369 | ||
2370 | /* Check that argument REL_ARGNO is an integer constant expression that | |
2371 | has the value VALUE0 or VALUE1. REL_ARGNO counts from the end of the | |
2372 | predication arguments. */ | |
2373 | bool | |
2374 | function_checker::require_immediate_either_or (unsigned int rel_argno, | |
2375 | HOST_WIDE_INT value0, | |
2376 | HOST_WIDE_INT value1) | |
2377 | { | |
2378 | unsigned int argno = m_base_arg + rel_argno; | |
2379 | if (!argument_exists_p (argno)) | |
2380 | return true; | |
2381 | ||
2382 | HOST_WIDE_INT actual; | |
2383 | if (!require_immediate (argno, actual)) | |
2384 | return false; | |
2385 | ||
2386 | if (actual != value0 && actual != value1) | |
2387 | { | |
2388 | report_neither_nor (location, fndecl, argno, actual, 90, 270); | |
2389 | return false; | |
2390 | } | |
2391 | ||
2392 | return true; | |
2393 | } | |
2394 | ||
2395 | /* Check that argument REL_ARGNO is an integer constant expression that has | |
2396 | a valid value for enumeration type TYPE. REL_ARGNO counts from the end | |
2397 | of the predication arguments. */ | |
2398 | bool | |
2399 | function_checker::require_immediate_enum (unsigned int rel_argno, tree type) | |
2400 | { | |
2401 | unsigned int argno = m_base_arg + rel_argno; | |
2402 | if (!argument_exists_p (argno)) | |
2403 | return true; | |
2404 | ||
2405 | HOST_WIDE_INT actual; | |
2406 | if (!require_immediate (argno, actual)) | |
2407 | return false; | |
2408 | ||
2409 | for (tree entry = TYPE_VALUES (type); entry; entry = TREE_CHAIN (entry)) | |
2410 | { | |
2411 | /* The value is an INTEGER_CST for C and a CONST_DECL wrapper | |
2412 | around an INTEGER_CST for C++. */ | |
2413 | tree value = TREE_VALUE (entry); | |
2414 | if (TREE_CODE (value) == CONST_DECL) | |
2415 | value = DECL_INITIAL (value); | |
2416 | if (wi::to_widest (value) == actual) | |
2417 | return true; | |
2418 | } | |
2419 | ||
2420 | report_not_enum (location, fndecl, argno, actual, type); | |
2421 | return false; | |
2422 | } | |
2423 | ||
2424 | /* Check that argument REL_ARGNO is suitable for indexing argument | |
2425 | REL_ARGNO - 1, in groups of GROUP_SIZE elements. REL_ARGNO counts | |
2426 | from the end of the predication arguments. */ | |
2427 | bool | |
2428 | function_checker::require_immediate_lane_index (unsigned int rel_argno, | |
2429 | unsigned int group_size) | |
2430 | { | |
2431 | unsigned int argno = m_base_arg + rel_argno; | |
2432 | if (!argument_exists_p (argno)) | |
2433 | return true; | |
2434 | ||
2435 | /* Get the type of the previous argument. tree_argument_type wants a | |
2436 | 1-based number, whereas ARGNO is 0-based. */ | |
2437 | machine_mode mode = TYPE_MODE (type_argument_type (m_fntype, argno)); | |
2438 | gcc_assert (VECTOR_MODE_P (mode)); | |
2439 | unsigned int nlanes = 128 / (group_size * GET_MODE_UNIT_BITSIZE (mode)); | |
2440 | return require_immediate_range (rel_argno, 0, nlanes - 1); | |
2441 | } | |
2442 | ||
2443 | /* Check that argument REL_ARGNO is an integer constant expression that | |
2444 | has one of the given values. */ | |
2445 | bool | |
2446 | function_checker::require_immediate_one_of (unsigned int rel_argno, | |
2447 | HOST_WIDE_INT value0, | |
2448 | HOST_WIDE_INT value1, | |
2449 | HOST_WIDE_INT value2, | |
2450 | HOST_WIDE_INT value3) | |
2451 | { | |
2452 | unsigned int argno = m_base_arg + rel_argno; | |
2453 | if (!argument_exists_p (argno)) | |
2454 | return true; | |
2455 | ||
2456 | HOST_WIDE_INT actual; | |
2457 | if (!require_immediate (argno, actual)) | |
2458 | return false; | |
2459 | ||
2460 | if (actual != value0 | |
2461 | && actual != value1 | |
2462 | && actual != value2 | |
2463 | && actual != value3) | |
2464 | { | |
2465 | report_not_one_of (location, fndecl, argno, actual, | |
2466 | value0, value1, value2, value3); | |
2467 | return false; | |
2468 | } | |
2469 | ||
2470 | return true; | |
2471 | } | |
2472 | ||
2473 | /* Check that argument REL_ARGNO is an integer constant expression in the | |
2474 | range [MIN, MAX]. REL_ARGNO counts from the end of the predication | |
2475 | arguments. */ | |
2476 | bool | |
2477 | function_checker::require_immediate_range (unsigned int rel_argno, | |
2478 | HOST_WIDE_INT min, | |
2479 | HOST_WIDE_INT max) | |
2480 | { | |
2481 | unsigned int argno = m_base_arg + rel_argno; | |
2482 | if (!argument_exists_p (argno)) | |
2483 | return true; | |
2484 | ||
2485 | /* Required because of the tree_to_uhwi -> HOST_WIDE_INT conversion | |
2486 | in require_immediate. */ | |
2487 | gcc_assert (min >= 0 && min <= max); | |
2488 | HOST_WIDE_INT actual; | |
2489 | if (!require_immediate (argno, actual)) | |
2490 | return false; | |
2491 | ||
2492 | if (!IN_RANGE (actual, min, max)) | |
2493 | { | |
2494 | report_out_of_range (location, fndecl, argno, actual, min, max); | |
2495 | return false; | |
2496 | } | |
2497 | ||
2498 | return true; | |
2499 | } | |
2500 | ||
2501 | /* Perform semantic checks on the call. Return true if the call is valid, | |
2502 | otherwise report a suitable error. */ | |
2503 | bool | |
2504 | function_checker::check () | |
2505 | { | |
2506 | function_args_iterator iter; | |
2507 | tree type; | |
2508 | unsigned int i = 0; | |
2509 | FOREACH_FUNCTION_ARGS (m_fntype, type, iter) | |
2510 | { | |
2511 | if (type == void_type_node || i >= m_nargs) | |
2512 | break; | |
2513 | ||
2514 | if (i >= m_base_arg | |
2515 | && TREE_CODE (type) == ENUMERAL_TYPE | |
2516 | && !require_immediate_enum (i - m_base_arg, type)) | |
2517 | return false; | |
2518 | ||
2519 | i += 1; | |
2520 | } | |
2521 | ||
2522 | return shape->check (*this); | |
2523 | } | |
2524 | ||
2525 | gimple_folder::gimple_folder (const function_instance &instance, tree fndecl, | |
2526 | gimple_stmt_iterator *gsi_in, gcall *call_in) | |
2527 | : function_call_info (gimple_location (call_in), instance, fndecl), | |
2528 | gsi (gsi_in), call (call_in), lhs (gimple_call_lhs (call_in)) | |
2529 | { | |
2530 | } | |
2531 | ||
0435b10d RS |
2532 | /* VALUE might be a vector of type VECTYPE or a single scalar element. |
2533 | Duplicate it into a vector of type VECTYPE in the latter case, adding any | |
2534 | new statements to STMTS. */ | |
2535 | tree | |
2536 | gimple_folder::force_vector (gimple_seq &stmts, tree vectype, tree value) | |
2537 | { | |
2538 | if (!VECTOR_TYPE_P (TREE_TYPE (value))) | |
2539 | value = gimple_build_vector_from_val (&stmts, vectype, value); | |
2540 | return value; | |
2541 | } | |
2542 | ||
624d0f07 RS |
2543 | /* Convert predicate argument ARGNO so that it has the type appropriate for |
2544 | an operation on VECTYPE. Add any new statements to STMTS. */ | |
2545 | tree | |
2546 | gimple_folder::convert_pred (gimple_seq &stmts, tree vectype, | |
2547 | unsigned int argno) | |
2548 | { | |
624d0f07 | 2549 | tree pred = gimple_call_arg (call, argno); |
482b2b43 RS |
2550 | if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred)), |
2551 | TYPE_VECTOR_SUBPARTS (vectype))) | |
2552 | return pred; | |
2553 | ||
2554 | return gimple_build (&stmts, VIEW_CONVERT_EXPR, | |
2555 | truth_type_for (vectype), pred); | |
624d0f07 RS |
2556 | } |
2557 | ||
2558 | /* Return a pointer to the address in a contiguous load or store, | |
2559 | given that each memory vector has type VECTYPE. Add any new | |
2560 | statements to STMTS. */ | |
2561 | tree | |
2562 | gimple_folder::fold_contiguous_base (gimple_seq &stmts, tree vectype) | |
2563 | { | |
2564 | tree base = gimple_call_arg (call, 1); | |
2565 | if (mode_suffix_id == MODE_vnum) | |
2566 | { | |
2567 | tree offset = gimple_call_arg (call, 2); | |
2568 | offset = gimple_convert (&stmts, sizetype, offset); | |
2569 | offset = gimple_build (&stmts, MULT_EXPR, sizetype, offset, | |
2570 | TYPE_SIZE_UNIT (vectype)); | |
2571 | base = gimple_build (&stmts, POINTER_PLUS_EXPR, TREE_TYPE (base), | |
2572 | base, offset); | |
2573 | } | |
2574 | return base; | |
2575 | } | |
2576 | ||
2577 | /* Return the alignment and TBAA argument to an internal load or store | |
2578 | function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses | |
2579 | memory elements of type TYPE. */ | |
2580 | tree | |
2581 | gimple_folder::load_store_cookie (tree type) | |
2582 | { | |
2583 | return build_int_cst (build_pointer_type (type), TYPE_ALIGN_UNIT (type)); | |
2584 | } | |
2585 | ||
0a09a948 RS |
2586 | /* Fold the call to a call to INSTANCE, with the same arguments. */ |
2587 | gimple * | |
2588 | gimple_folder::redirect_call (const function_instance &instance) | |
2589 | { | |
2590 | registered_function *rfn | |
2591 | = function_table->find_with_hash (instance, instance.hash ()); | |
2592 | if (!rfn) | |
2593 | return NULL; | |
2594 | ||
2595 | gimple_call_set_fndecl (call, rfn->decl); | |
2596 | return call; | |
2597 | } | |
2598 | ||
624d0f07 RS |
2599 | /* Fold the call to a PTRUE, taking the element size from type suffix 0. */ |
2600 | gimple * | |
2601 | gimple_folder::fold_to_ptrue () | |
2602 | { | |
2603 | tree svbool_type = TREE_TYPE (lhs); | |
2604 | tree bool_type = TREE_TYPE (svbool_type); | |
2605 | unsigned int element_bytes = type_suffix (0).element_bytes; | |
2606 | ||
2607 | /* The return type is svbool_t for all type suffixes, thus for b8 we | |
2608 | want { 1, 1, 1, 1, ... }, for b16 we want { 1, 0, 1, 0, ... }, etc. */ | |
2609 | tree_vector_builder builder (svbool_type, element_bytes, 1); | |
2610 | builder.quick_push (build_all_ones_cst (bool_type)); | |
2611 | for (unsigned int i = 1; i < element_bytes; ++i) | |
2612 | builder.quick_push (build_zero_cst (bool_type)); | |
2613 | return gimple_build_assign (lhs, builder.build ()); | |
2614 | } | |
2615 | ||
2616 | /* Fold the call to a PFALSE. */ | |
2617 | gimple * | |
2618 | gimple_folder::fold_to_pfalse () | |
2619 | { | |
2620 | return gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs))); | |
2621 | } | |
2622 | ||
2623 | /* Fold an operation to a constant predicate in which the first VL | |
2624 | elements are set and the rest are clear. Take the element size | |
2625 | from type suffix 0. */ | |
2626 | gimple * | |
2627 | gimple_folder::fold_to_vl_pred (unsigned int vl) | |
2628 | { | |
2629 | tree vectype = TREE_TYPE (lhs); | |
2630 | tree element_type = TREE_TYPE (vectype); | |
2631 | tree minus_one = build_all_ones_cst (element_type); | |
2632 | tree zero = build_zero_cst (element_type); | |
2633 | unsigned int element_bytes = type_suffix (0).element_bytes; | |
2634 | ||
2635 | /* Construct COUNT elements that contain the ptrue followed by | |
2636 | a repeating sequence of COUNT elements. */ | |
2637 | unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype)); | |
2638 | gcc_assert (vl * element_bytes <= count); | |
2639 | tree_vector_builder builder (vectype, count, 2); | |
2640 | for (unsigned int i = 0; i < count * 2; ++i) | |
2641 | { | |
2642 | bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes; | |
2643 | builder.quick_push (bit ? minus_one : zero); | |
2644 | } | |
2645 | return gimple_build_assign (lhs, builder.build ()); | |
2646 | } | |
2647 | ||
2648 | /* Try to fold the call. Return the new statement on success and null | |
2649 | on failure. */ | |
2650 | gimple * | |
2651 | gimple_folder::fold () | |
2652 | { | |
2653 | /* Don't fold anything when SVE is disabled; emit an error during | |
2654 | expansion instead. */ | |
2655 | if (!TARGET_SVE) | |
2656 | return NULL; | |
2657 | ||
2658 | /* Punt if the function has a return type and no result location is | |
2659 | provided. The attributes should allow target-independent code to | |
2660 | remove the calls if appropriate. */ | |
2661 | if (!lhs && TREE_TYPE (gimple_call_fntype (call)) != void_type_node) | |
2662 | return NULL; | |
2663 | ||
2664 | return base->fold (*this); | |
2665 | } | |
2666 | ||
2667 | function_expander::function_expander (const function_instance &instance, | |
2668 | tree fndecl, tree call_expr_in, | |
2669 | rtx possible_target_in) | |
2670 | : function_call_info (EXPR_LOCATION (call_expr_in), instance, fndecl), | |
2671 | call_expr (call_expr_in), possible_target (possible_target_in) | |
2672 | { | |
2673 | } | |
2674 | ||
2675 | /* Return the handler of direct optab OP for type suffix SUFFIX_I. */ | |
2676 | insn_code | |
2677 | function_expander::direct_optab_handler (optab op, unsigned int suffix_i) | |
2678 | { | |
2679 | return ::direct_optab_handler (op, vector_mode (suffix_i)); | |
2680 | } | |
2681 | ||
2682 | /* Choose between signed and unsigned direct optabs SIGNED_OP and | |
2683 | UNSIGNED_OP based on the signedness of type suffix SUFFIX_I, then | |
2684 | pick the appropriate optab handler for the mode. Use MODE as the | |
2685 | mode if given, otherwise use the mode of type suffix SUFFIX_I. */ | |
2686 | insn_code | |
2687 | function_expander::direct_optab_handler_for_sign (optab signed_op, | |
2688 | optab unsigned_op, | |
2689 | unsigned int suffix_i, | |
2690 | machine_mode mode) | |
2691 | { | |
2692 | if (mode == VOIDmode) | |
2693 | mode = vector_mode (suffix_i); | |
2694 | optab op = type_suffix (suffix_i).unsigned_p ? unsigned_op : signed_op; | |
2695 | return ::direct_optab_handler (op, mode); | |
2696 | } | |
2697 | ||
2698 | /* Return true if X overlaps any input. */ | |
2699 | bool | |
2700 | function_expander::overlaps_input_p (rtx x) | |
2701 | { | |
2702 | for (unsigned int i = 0; i < args.length (); ++i) | |
2703 | if (reg_overlap_mentioned_p (x, args[i])) | |
2704 | return true; | |
2705 | return false; | |
2706 | } | |
2707 | ||
fb15e2ba RS |
2708 | /* Convert ptr_mode value X to Pmode. */ |
2709 | rtx | |
2710 | function_expander::convert_to_pmode (rtx x) | |
2711 | { | |
2712 | if (ptr_mode == SImode) | |
2713 | x = simplify_gen_unary (ZERO_EXTEND, DImode, x, SImode); | |
2714 | return x; | |
2715 | } | |
2716 | ||
624d0f07 RS |
2717 | /* Return the base address for a contiguous load or store function. |
2718 | MEM_MODE is the mode of the addressed memory. */ | |
2719 | rtx | |
2720 | function_expander::get_contiguous_base (machine_mode mem_mode) | |
2721 | { | |
fb15e2ba | 2722 | rtx base = convert_to_pmode (args[1]); |
624d0f07 RS |
2723 | if (mode_suffix_id == MODE_vnum) |
2724 | { | |
2725 | /* Use the size of the memory mode for extending loads and truncating | |
2726 | stores. Use the size of a full vector for non-extending loads | |
2727 | and non-truncating stores (including svld[234] and svst[234]). */ | |
2728 | poly_int64 size = ordered_min (GET_MODE_SIZE (mem_mode), | |
2729 | BYTES_PER_SVE_VECTOR); | |
2730 | rtx offset = gen_int_mode (size, Pmode); | |
2731 | offset = simplify_gen_binary (MULT, Pmode, args[2], offset); | |
2732 | base = simplify_gen_binary (PLUS, Pmode, base, offset); | |
2733 | } | |
2734 | return base; | |
2735 | } | |
2736 | ||
2737 | /* For a function that does the equivalent of: | |
2738 | ||
2739 | OUTPUT = COND ? FN (INPUTS) : FALLBACK; | |
2740 | ||
2741 | return the value of FALLBACK. | |
2742 | ||
2743 | MODE is the mode of OUTPUT. NOPS is the number of operands in INPUTS. | |
2744 | MERGE_ARGNO is the argument that provides FALLBACK for _m functions, | |
2745 | or DEFAULT_MERGE_ARGNO if we should apply the usual rules. | |
2746 | ||
2747 | ARGNO is the caller's index into args. If the returned value is | |
2748 | argument 0 (as for unary _m operations), increment ARGNO past the | |
2749 | returned argument. */ | |
2750 | rtx | |
2751 | function_expander::get_fallback_value (machine_mode mode, unsigned int nops, | |
2752 | unsigned int merge_argno, | |
2753 | unsigned int &argno) | |
2754 | { | |
2755 | if (pred == PRED_z) | |
2756 | return CONST0_RTX (mode); | |
2757 | ||
2758 | gcc_assert (pred == PRED_m || pred == PRED_x); | |
2759 | if (merge_argno == DEFAULT_MERGE_ARGNO) | |
2760 | merge_argno = nops == 1 && pred == PRED_m ? 0 : 1; | |
2761 | ||
2762 | if (merge_argno == 0) | |
2763 | return args[argno++]; | |
2764 | ||
2765 | return args[merge_argno]; | |
2766 | } | |
2767 | ||
2768 | /* Return a REG rtx that can be used for the result of the function, | |
2769 | using the preferred target if suitable. */ | |
2770 | rtx | |
2771 | function_expander::get_reg_target () | |
2772 | { | |
2773 | machine_mode target_mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl))); | |
2774 | if (!possible_target || GET_MODE (possible_target) != target_mode) | |
2775 | possible_target = gen_reg_rtx (target_mode); | |
2776 | return possible_target; | |
2777 | } | |
2778 | ||
2779 | /* As for get_reg_target, but make sure that the returned REG does not | |
2780 | overlap any inputs. */ | |
2781 | rtx | |
2782 | function_expander::get_nonoverlapping_reg_target () | |
2783 | { | |
2784 | if (possible_target && overlaps_input_p (possible_target)) | |
2785 | possible_target = NULL_RTX; | |
2786 | return get_reg_target (); | |
2787 | } | |
2788 | ||
2789 | /* Add an output operand to the instruction we're building, which has | |
2790 | code ICODE. Bind the output to the preferred target rtx if possible. */ | |
2791 | void | |
2792 | function_expander::add_output_operand (insn_code icode) | |
2793 | { | |
2794 | unsigned int opno = m_ops.length (); | |
2795 | machine_mode mode = insn_data[icode].operand[opno].mode; | |
cb3874dc | 2796 | m_ops.safe_grow (opno + 1, true); |
624d0f07 RS |
2797 | create_output_operand (&m_ops.last (), possible_target, mode); |
2798 | } | |
2799 | ||
2800 | /* Add an input operand to the instruction we're building, which has | |
2801 | code ICODE. Calculate the value of the operand as follows: | |
2802 | ||
2803 | - If the operand is a vector and X is not, broadcast X to fill a | |
2804 | vector of the appropriate mode. | |
2805 | ||
2806 | - Otherwise, if the operand is a predicate, coerce X to have the | |
2807 | mode that the instruction expects. In this case X is known to be | |
2808 | VNx16BImode (the mode of svbool_t). | |
2809 | ||
2810 | - Otherwise use X directly. The expand machinery checks that X has | |
2811 | the right mode for the instruction. */ | |
2812 | void | |
2813 | function_expander::add_input_operand (insn_code icode, rtx x) | |
2814 | { | |
2815 | unsigned int opno = m_ops.length (); | |
2816 | const insn_operand_data &operand = insn_data[icode].operand[opno]; | |
2817 | machine_mode mode = operand.mode; | |
2818 | if (mode == VOIDmode) | |
2819 | { | |
2820 | /* The only allowable use of VOIDmode is the wildcard | |
2821 | aarch64_any_register_operand, which is used to avoid | |
2822 | combinatorial explosion in the reinterpret patterns. */ | |
2823 | gcc_assert (operand.predicate == aarch64_any_register_operand); | |
2824 | mode = GET_MODE (x); | |
2825 | } | |
2826 | else if (!VECTOR_MODE_P (GET_MODE (x)) && VECTOR_MODE_P (mode)) | |
2827 | x = expand_vector_broadcast (mode, x); | |
2828 | else if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL) | |
2829 | { | |
2830 | gcc_assert (GET_MODE (x) == VNx16BImode); | |
2831 | x = gen_lowpart (mode, x); | |
2832 | } | |
cb3874dc | 2833 | m_ops.safe_grow (m_ops.length () + 1, true); |
624d0f07 RS |
2834 | create_input_operand (&m_ops.last (), x, mode); |
2835 | } | |
2836 | ||
2837 | /* Add an integer operand with value X to the instruction. */ | |
2838 | void | |
2839 | function_expander::add_integer_operand (HOST_WIDE_INT x) | |
2840 | { | |
cb3874dc | 2841 | m_ops.safe_grow (m_ops.length () + 1, true); |
624d0f07 RS |
2842 | create_integer_operand (&m_ops.last (), x); |
2843 | } | |
2844 | ||
2845 | /* Add a memory operand with mode MODE and address ADDR. */ | |
2846 | void | |
2847 | function_expander::add_mem_operand (machine_mode mode, rtx addr) | |
2848 | { | |
9ceec73f MM |
2849 | /* Exception for OImode for the ld1ro intrinsics. |
2850 | They act on 256 bit octaword data, and it's just easier to use a scalar | |
2851 | mode to represent that than add a new vector mode solely for the purpose | |
2852 | of this intrinsic. */ | |
2853 | gcc_assert (VECTOR_MODE_P (mode) || mode == OImode); | |
624d0f07 RS |
2854 | rtx mem = gen_rtx_MEM (mode, memory_address (mode, addr)); |
2855 | /* The memory is only guaranteed to be element-aligned. */ | |
2856 | set_mem_align (mem, GET_MODE_ALIGNMENT (GET_MODE_INNER (mode))); | |
2857 | add_fixed_operand (mem); | |
2858 | } | |
2859 | ||
2860 | /* Add an address operand with value X. The static operand data says | |
2861 | what mode and form the address must have. */ | |
2862 | void | |
2863 | function_expander::add_address_operand (rtx x) | |
2864 | { | |
cb3874dc | 2865 | m_ops.safe_grow (m_ops.length () + 1, true); |
624d0f07 RS |
2866 | create_address_operand (&m_ops.last (), x); |
2867 | } | |
2868 | ||
2869 | /* Add an operand that must be X. The only way of legitimizing an | |
2870 | invalid X is to reload the address of a MEM. */ | |
2871 | void | |
2872 | function_expander::add_fixed_operand (rtx x) | |
2873 | { | |
cb3874dc | 2874 | m_ops.safe_grow (m_ops.length () + 1, true); |
624d0f07 RS |
2875 | create_fixed_operand (&m_ops.last (), x); |
2876 | } | |
2877 | ||
2878 | /* Generate instruction ICODE, given that its operands have already | |
2879 | been added to M_OPS. Return the value of the first operand. */ | |
2880 | rtx | |
2881 | function_expander::generate_insn (insn_code icode) | |
2882 | { | |
2883 | expand_insn (icode, m_ops.length (), m_ops.address ()); | |
2884 | return function_returns_void_p () ? const0_rtx : m_ops[0].value; | |
2885 | } | |
2886 | ||
2887 | /* Convert the arguments to a gather/scatter function into the | |
2888 | associated md operands. Argument ARGNO is the scalar or vector base and | |
2889 | argument ARGNO + 1 is the scalar or vector displacement (if applicable). | |
2890 | The md pattern expects: | |
2891 | ||
2892 | - a scalar base | |
2893 | - a vector displacement | |
0a09a948 RS |
2894 | |
2895 | If SCALED_P is true, it also expects: | |
2896 | ||
624d0f07 | 2897 | - a const_int that is 1 if the displacement is zero-extended from 32 bits |
0a09a948 RS |
2898 | - a scaling multiplier (1 for bytes, 2 for .h indices, etc.). |
2899 | ||
2900 | If SCALED_P is false, the displacement is implicitly zero-extended | |
2901 | and the scaling multiplier is implicitly 1. */ | |
624d0f07 | 2902 | void |
0a09a948 RS |
2903 | function_expander::prepare_gather_address_operands (unsigned int argno, |
2904 | bool scaled_p) | |
624d0f07 RS |
2905 | { |
2906 | machine_mode mem_mode = memory_vector_mode (); | |
2907 | tree vector_type = base_vector_type (); | |
2908 | units_index units = displacement_units (); | |
0a09a948 | 2909 | int shift_idx = -1; |
624d0f07 RS |
2910 | if (units == UNITS_none) |
2911 | { | |
2912 | /* Vector base, no displacement. Convert to an integer zero base | |
2913 | and a vector byte offset. */ | |
2914 | args.quick_insert (argno, const0_rtx); | |
2915 | units = UNITS_bytes; | |
2916 | } | |
2917 | else if (vector_type) | |
2918 | { | |
2919 | /* Vector base, scalar displacement. Convert to a scalar base and | |
2920 | a vector byte offset. */ | |
2921 | std::swap (args[argno], args[argno + 1]); | |
2922 | if (units == UNITS_elements) | |
0a09a948 | 2923 | shift_idx = argno; |
624d0f07 RS |
2924 | } |
2925 | else | |
2926 | { | |
0a09a948 RS |
2927 | /* Scalar base, vector displacement. This is the order that the md |
2928 | pattern wants. */ | |
fb15e2ba | 2929 | args[argno] = convert_to_pmode (args[argno]); |
624d0f07 | 2930 | vector_type = displacement_vector_type (); |
0a09a948 RS |
2931 | if (units == UNITS_elements && !scaled_p) |
2932 | shift_idx = argno + 1; | |
624d0f07 RS |
2933 | } |
2934 | tree scalar_displacement_type = TREE_TYPE (vector_type); | |
2935 | ||
0a09a948 RS |
2936 | if (shift_idx >= 0) |
2937 | { | |
2938 | machine_mode arg_mode = GET_MODE (args[shift_idx]); | |
2939 | if (arg_mode == VOIDmode) | |
2940 | arg_mode = DImode; | |
2941 | unsigned int elt_bytes = GET_MODE_UNIT_SIZE (mem_mode); | |
2942 | rtx shift = gen_int_mode (exact_log2 (elt_bytes), DImode); | |
2943 | args[shift_idx] = simplify_gen_binary (ASHIFT, arg_mode, | |
2944 | args[shift_idx], shift); | |
2945 | units = UNITS_bytes; | |
2946 | } | |
2947 | ||
2948 | bool uxtw_p = (TYPE_PRECISION (scalar_displacement_type) == 64 | |
2949 | || TYPE_UNSIGNED (scalar_displacement_type)); | |
624d0f07 RS |
2950 | unsigned int scale = (units == UNITS_bytes |
2951 | ? 1 : GET_MODE_UNIT_SIZE (mem_mode)); | |
2952 | ||
0a09a948 RS |
2953 | if (scaled_p) |
2954 | { | |
2955 | args.quick_insert (argno + 2, GEN_INT (uxtw_p)); | |
2956 | args.quick_insert (argno + 3, GEN_INT (scale)); | |
2957 | } | |
2958 | else | |
2959 | gcc_assert (uxtw_p && scale == 1); | |
624d0f07 RS |
2960 | } |
2961 | ||
2962 | /* The final argument is an immediate svprfop value. Add two fake arguments | |
2963 | to represent the rw and locality operands of a PREFETCH rtx. */ | |
2964 | void | |
2965 | function_expander::prepare_prefetch_operands () | |
2966 | { | |
2967 | unsigned int prfop = INTVAL (args.last ()); | |
2968 | /* Bit 3 of the prfop selects stores over loads. */ | |
2969 | args.quick_push (GEN_INT ((prfop & 8) != 0)); | |
2970 | /* Bits 1 and 2 specify the locality; 0-based for svprfop but | |
2971 | 1-based for PREFETCH. */ | |
2972 | args.quick_push (GEN_INT (((prfop >> 1) & 3) + 1)); | |
2973 | } | |
2974 | ||
2975 | /* Add a dummy argument to indicate whether predicate argument ARGNO | |
2976 | is all-true when interpreted in mode PRED_MODE. The hint goes | |
2977 | immediately after ARGNO. */ | |
2978 | void | |
2979 | function_expander::add_ptrue_hint (unsigned int argno, machine_mode pred_mode) | |
2980 | { | |
2981 | rtx pred = gen_lowpart (pred_mode, args[argno]); | |
2982 | int hint = (pred == CONSTM1_RTX (pred_mode) | |
2983 | ? SVE_KNOWN_PTRUE : SVE_MAYBE_NOT_PTRUE); | |
2984 | args.quick_insert (argno + 1, gen_int_mode (hint, SImode)); | |
2985 | } | |
2986 | ||
2987 | /* Rotate inputs args[START:END] one position to the left, so that | |
2988 | args[START] becomes args[END - 1]. */ | |
2989 | void | |
2990 | function_expander::rotate_inputs_left (unsigned int start, unsigned int end) | |
2991 | { | |
2992 | rtx new_last = args[start]; | |
2993 | for (unsigned int i = start; i < end - 1; ++i) | |
2994 | args[i] = args[i + 1]; | |
2995 | args[end - 1] = new_last; | |
2996 | } | |
2997 | ||
2998 | /* Return true if the negation of argument ARGNO can be folded away, | |
2999 | replacing it with the negated value if so. MODE is the associated | |
3000 | vector mode, but the argument could be a single element. The main | |
3001 | case this handles is constant arguments. */ | |
3002 | bool | |
3003 | function_expander::try_negating_argument (unsigned int argno, | |
3004 | machine_mode mode) | |
3005 | { | |
3006 | rtx x = args[argno]; | |
3007 | if (!VECTOR_MODE_P (GET_MODE (x))) | |
3008 | mode = GET_MODE_INNER (mode); | |
3009 | ||
3010 | x = simplify_unary_operation (NEG, mode, x, mode); | |
3011 | if (!x) | |
3012 | return false; | |
3013 | ||
3014 | args[argno] = x; | |
3015 | return true; | |
3016 | } | |
3017 | ||
3018 | /* Implement the call using instruction ICODE, with a 1:1 mapping between | |
3019 | arguments and input operands. */ | |
3020 | rtx | |
3021 | function_expander::use_exact_insn (insn_code icode) | |
3022 | { | |
3023 | unsigned int nops = insn_data[icode].n_operands; | |
3024 | if (!function_returns_void_p ()) | |
3025 | { | |
3026 | add_output_operand (icode); | |
3027 | nops -= 1; | |
3028 | } | |
3029 | for (unsigned int i = 0; i < nops; ++i) | |
3030 | add_input_operand (icode, args[i]); | |
3031 | return generate_insn (icode); | |
3032 | } | |
3033 | ||
3034 | /* Implement the call using instruction ICODE, which does not use a | |
3035 | governing predicate. We must therefore drop the GP from an _x call. */ | |
3036 | rtx | |
3037 | function_expander::use_unpred_insn (insn_code icode) | |
3038 | { | |
3039 | /* We can't drop the predicate for _z and _m. */ | |
3040 | gcc_assert (pred == PRED_x || pred == PRED_none); | |
3041 | /* Discount the output operand. */ | |
3042 | unsigned int nops = insn_data[icode].n_operands - 1; | |
3043 | /* Drop the predicate argument in the case of _x predication. */ | |
3044 | unsigned int bias = (pred == PRED_x ? 1 : 0); | |
3045 | unsigned int i = 0; | |
3046 | ||
3047 | add_output_operand (icode); | |
3048 | for (; i < nops; ++i) | |
3049 | add_input_operand (icode, args[i + bias]); | |
3050 | ||
3051 | return generate_insn (icode); | |
3052 | } | |
3053 | ||
3054 | /* Implement the call using instruction ICODE, which is a predicated | |
3055 | operation that returns arbitrary values for inactive lanes. */ | |
3056 | rtx | |
3057 | function_expander::use_pred_x_insn (insn_code icode) | |
3058 | { | |
3059 | /* At present we never need to handle PRED_none, which would involve | |
3060 | creating a new predicate rather than using one supplied by the user. */ | |
3061 | gcc_assert (pred == PRED_x); | |
3062 | /* Discount the output operand. */ | |
3063 | unsigned int nops = args.length () - 1; | |
3064 | ||
3065 | bool has_float_operand_p = FLOAT_MODE_P (insn_data[icode].operand[0].mode); | |
3066 | ||
3067 | /* Add the normal operands. */ | |
3068 | add_output_operand (icode); | |
3069 | add_input_operand (icode, args[0]); | |
3070 | for (unsigned int i = 0; i < nops; ++i) | |
3071 | { | |
3072 | add_input_operand (icode, args[i + 1]); | |
3073 | if (FLOAT_MODE_P (GET_MODE (args[i + 1]))) | |
3074 | has_float_operand_p = true; | |
3075 | } | |
3076 | ||
3077 | if (has_float_operand_p) | |
3078 | { | |
3079 | /* Add a flag that indicates whether unpredicated instructions | |
3080 | are allowed. */ | |
3081 | rtx pred = m_ops[1].value; | |
3082 | if (flag_trapping_math && pred != CONST1_RTX (GET_MODE (pred))) | |
3083 | add_integer_operand (SVE_STRICT_GP); | |
3084 | else | |
3085 | add_integer_operand (SVE_RELAXED_GP); | |
3086 | } | |
3087 | ||
3088 | return generate_insn (icode); | |
3089 | } | |
3090 | ||
3091 | /* Implement the call using instruction ICODE, which does the equivalent of: | |
3092 | ||
3093 | OUTPUT = COND ? FN (INPUTS) : FALLBACK; | |
3094 | ||
3095 | The instruction operands are in the order above: OUTPUT, COND, INPUTS | |
3096 | and FALLBACK. MERGE_ARGNO is the argument that provides FALLBACK for _m | |
3097 | functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ | |
3098 | rtx | |
3099 | function_expander::use_cond_insn (insn_code icode, unsigned int merge_argno) | |
3100 | { | |
3101 | /* At present we never need to handle PRED_none, which would involve | |
3102 | creating a new predicate rather than using one supplied by the user. */ | |
3103 | gcc_assert (pred != PRED_none); | |
3104 | /* Discount the output, predicate and fallback value. */ | |
3105 | unsigned int nops = insn_data[icode].n_operands - 3; | |
3106 | machine_mode mode = insn_data[icode].operand[0].mode; | |
3107 | ||
3108 | unsigned int opno = 0; | |
3109 | rtx fallback_arg = get_fallback_value (mode, nops, merge_argno, opno); | |
3110 | rtx pred = args[opno++]; | |
3111 | ||
3112 | add_output_operand (icode); | |
3113 | add_input_operand (icode, pred); | |
3114 | for (unsigned int i = 0; i < nops; ++i) | |
3115 | add_input_operand (icode, args[opno + i]); | |
3116 | add_input_operand (icode, fallback_arg); | |
3117 | return generate_insn (icode); | |
3118 | } | |
3119 | ||
3120 | /* Implement the call using instruction ICODE, which is a select-like | |
3121 | operation with the following operands: | |
3122 | ||
3123 | 0: output | |
3124 | 1: true value | |
3125 | 2: false value | |
3126 | 3: predicate | |
3127 | ||
3128 | MERGE_ARGNO is the argument that provides the "false" value for _m | |
3129 | functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ | |
3130 | rtx | |
3131 | function_expander::use_vcond_mask_insn (insn_code icode, | |
3132 | unsigned int merge_argno) | |
3133 | { | |
3134 | machine_mode mode = vector_mode (0); | |
3135 | ||
3136 | unsigned int opno = 0; | |
3137 | rtx false_arg = get_fallback_value (mode, 1, merge_argno, opno); | |
3138 | rtx pred_arg = args[opno++]; | |
3139 | rtx true_arg = args[opno++]; | |
3140 | ||
3141 | add_output_operand (icode); | |
3142 | add_input_operand (icode, true_arg); | |
3143 | add_input_operand (icode, false_arg); | |
3144 | add_input_operand (icode, pred_arg); | |
3145 | return generate_insn (icode); | |
3146 | } | |
3147 | ||
3148 | /* Implement the call using instruction ICODE, which loads memory operand 1 | |
217ccab8 RS |
3149 | into register operand 0 under the control of predicate operand 2. |
3150 | Extending loads have a further predicate (operand 3) that nominally | |
3151 | controls the extension. */ | |
624d0f07 RS |
3152 | rtx |
3153 | function_expander::use_contiguous_load_insn (insn_code icode) | |
3154 | { | |
3155 | machine_mode mem_mode = memory_vector_mode (); | |
3156 | ||
3157 | add_output_operand (icode); | |
3158 | add_mem_operand (mem_mode, get_contiguous_base (mem_mode)); | |
3159 | add_input_operand (icode, args[0]); | |
217ccab8 RS |
3160 | if (GET_MODE_UNIT_BITSIZE (mem_mode) < type_suffix (0).element_bits) |
3161 | add_input_operand (icode, CONSTM1_RTX (VNx16BImode)); | |
624d0f07 RS |
3162 | return generate_insn (icode); |
3163 | } | |
3164 | ||
3165 | /* Implement the call using instruction ICODE, which prefetches from | |
3166 | address operand 1 under the control of predicate operand 0. | |
3167 | Operands 2, 3 and 4 respectively specify the svprfop value, | |
3168 | the PREFETCH rw flag and the PREFETCH locality. */ | |
3169 | rtx | |
3170 | function_expander::use_contiguous_prefetch_insn (insn_code icode) | |
3171 | { | |
3172 | add_input_operand (icode, args[0]); | |
3173 | add_address_operand (get_contiguous_base (VNx16QImode)); | |
3174 | for (unsigned int i = args.length () - 3; i < args.length (); ++i) | |
3175 | add_input_operand (icode, args[i]); | |
3176 | return generate_insn (icode); | |
3177 | } | |
3178 | ||
3179 | /* Implement the call using instruction ICODE, which stores register operand 1 | |
3180 | into memory operand 0 under the control of predicate operand 2. */ | |
3181 | rtx | |
3182 | function_expander::use_contiguous_store_insn (insn_code icode) | |
3183 | { | |
3184 | machine_mode mem_mode = memory_vector_mode (); | |
3185 | ||
3186 | add_mem_operand (mem_mode, get_contiguous_base (mem_mode)); | |
3187 | add_input_operand (icode, args.last ()); | |
3188 | add_input_operand (icode, args[0]); | |
3189 | return generate_insn (icode); | |
3190 | } | |
3191 | ||
3192 | /* Implement the call using one of the following strategies, chosen in order: | |
3193 | ||
3194 | (1) "aarch64_pred_<optab><mode>_z" for PRED_z predicate functions | |
3195 | ||
3196 | (2) "aarch64_pred_<optab><mode>" for PRED_x functions | |
3197 | ||
3198 | (3) a normal unpredicated optab for PRED_none and PRED_x functions, | |
3199 | dropping the predicate in the latter case | |
3200 | ||
694e6b19 RS |
3201 | (4) an unpredicated "aarch64_sve_<code_optab><mode>" for PRED_none and |
3202 | PRED_x functions, again dropping the predicate for PRED_x | |
3203 | ||
3204 | (5) "cond_<optab><mode>" otherwise | |
624d0f07 RS |
3205 | |
3206 | where <optab> corresponds to: | |
3207 | ||
3208 | - CODE_FOR_SINT for signed integers | |
3209 | - CODE_FOR_UINT for unsigned integers | |
3210 | - UNSPEC_FOR_FP for floating-point values | |
3211 | ||
694e6b19 RS |
3212 | and where <code_optab> is like <optab>, but uses CODE_FOR_SINT instead |
3213 | of UNSPEC_FOR_FP for floating-point values. | |
3214 | ||
624d0f07 RS |
3215 | MERGE_ARGNO is the argument that provides the values of inactive lanes for |
3216 | _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ | |
3217 | rtx | |
3218 | function_expander::map_to_rtx_codes (rtx_code code_for_sint, | |
3219 | rtx_code code_for_uint, | |
3220 | int unspec_for_fp, | |
3221 | unsigned int merge_argno) | |
3222 | { | |
3223 | machine_mode mode = vector_mode (0); | |
3224 | rtx_code code = (type_suffix (0).unsigned_p ? code_for_uint : code_for_sint); | |
3225 | insn_code icode; | |
3226 | ||
3227 | /* Handle predicate logic operations, which always use _z predication. */ | |
3228 | if (type_suffix (0).tclass == TYPE_bool) | |
3229 | { | |
3230 | gcc_assert (pred == PRED_z && code_for_uint == code_for_sint); | |
3231 | return use_exact_insn (code_for_aarch64_pred_z (code, mode)); | |
3232 | } | |
3233 | ||
3234 | /* First try using UNSPEC_PRED_X patterns for _x predication, | |
3235 | if available. */ | |
3236 | if (pred == PRED_x) | |
3237 | { | |
3238 | if (type_suffix (0).integer_p) | |
3239 | icode = maybe_code_for_aarch64_pred (code, mode); | |
3240 | else | |
3241 | icode = maybe_code_for_aarch64_pred (unspec_for_fp, mode); | |
3242 | if (icode != CODE_FOR_nothing) | |
3243 | return use_pred_x_insn (icode); | |
3244 | } | |
3245 | ||
3246 | /* Otherwise expand PRED_none and PRED_x operations without a predicate. | |
3247 | Floating-point operations conventionally use the signed rtx code. */ | |
3248 | if (pred == PRED_none || pred == PRED_x) | |
694e6b19 RS |
3249 | { |
3250 | icode = direct_optab_handler (code_to_optab (code), 0); | |
3251 | if (icode == CODE_FOR_nothing) | |
3252 | icode = code_for_aarch64_sve (code, mode); | |
3253 | return use_unpred_insn (icode); | |
3254 | } | |
624d0f07 RS |
3255 | |
3256 | /* Don't use cond_*_optabs here, since not all codes have one yet. */ | |
3257 | if (type_suffix (0).integer_p) | |
3258 | icode = code_for_cond (code, mode); | |
3259 | else | |
3260 | icode = code_for_cond (unspec_for_fp, mode); | |
3261 | return use_cond_insn (icode, merge_argno); | |
3262 | } | |
3263 | ||
3264 | /* Implement the call using one of the following strategies, chosen in order: | |
3265 | ||
3266 | (1) "aarch64_pred_<optab><mode>" for PRED_x functions; this is a | |
3267 | predicated pattern | |
3268 | ||
3269 | (2) "aarch64_sve_<optab><mode>" for PRED_none and PRED_x functions; | |
3270 | this is an unpredicated pattern | |
3271 | ||
3272 | (3) "cond_<optab><mode>" otherwise | |
3273 | ||
3274 | where <optab> corresponds to: | |
3275 | ||
3276 | - UNSPEC_FOR_SINT for signed integers | |
3277 | - UNSPEC_FOR_UINT for unsigned integers | |
3278 | - UNSPEC_FOR_FP for floating-point values | |
3279 | ||
3280 | MERGE_ARGNO is the argument that provides the values of inactive lanes for | |
3281 | _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ | |
3282 | rtx | |
3283 | function_expander::map_to_unspecs (int unspec_for_sint, int unspec_for_uint, | |
3284 | int unspec_for_fp, unsigned int merge_argno) | |
3285 | { | |
3286 | machine_mode mode = vector_mode (0); | |
3287 | int unspec = (!type_suffix (0).integer_p ? unspec_for_fp | |
3288 | : type_suffix (0).unsigned_p ? unspec_for_uint | |
3289 | : unspec_for_sint); | |
3290 | ||
3291 | if (pred == PRED_x) | |
3292 | { | |
3293 | insn_code icode = maybe_code_for_aarch64_pred (unspec, mode); | |
3294 | if (icode != CODE_FOR_nothing) | |
3295 | return use_pred_x_insn (icode); | |
3296 | } | |
3297 | ||
3298 | if (pred == PRED_none || pred == PRED_x) | |
0a09a948 RS |
3299 | { |
3300 | insn_code icode = maybe_code_for_aarch64_sve (unspec, mode); | |
3301 | if (icode != CODE_FOR_nothing) | |
3302 | return use_unpred_insn (icode); | |
3303 | } | |
624d0f07 RS |
3304 | |
3305 | insn_code icode = code_for_cond (unspec, vector_mode (0)); | |
3306 | return use_cond_insn (icode, merge_argno); | |
3307 | } | |
3308 | ||
624d0f07 RS |
3309 | /* Expand the call and return its lhs. */ |
3310 | rtx | |
3311 | function_expander::expand () | |
3312 | { | |
3313 | unsigned int nargs = call_expr_nargs (call_expr); | |
3314 | args.reserve (nargs); | |
3315 | for (unsigned int i = 0; i < nargs; ++i) | |
3316 | args.quick_push (expand_normal (CALL_EXPR_ARG (call_expr, i))); | |
3317 | ||
3318 | return base->expand (*this); | |
3319 | } | |
3320 | ||
3321 | /* Register the built-in SVE ABI types, such as __SVBool_t. */ | |
3322 | static void | |
3323 | register_builtin_types () | |
3324 | { | |
3325 | #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \ | |
3326 | scalar_types[VECTOR_TYPE_ ## ACLE_NAME] = SCALAR_TYPE; | |
3327 | #include "aarch64-sve-builtins.def" | |
3328 | ||
3329 | for (unsigned int i = 0; i < NUM_VECTOR_TYPES; ++i) | |
3330 | { | |
3331 | tree eltype = scalar_types[i]; | |
3332 | tree vectype; | |
683e93d1 | 3333 | unsigned int num_zr = 0, num_pr = 0; |
624d0f07 RS |
3334 | if (eltype == boolean_type_node) |
3335 | { | |
0a0ef238 RS |
3336 | vectype = build_truth_vector_type_for_mode (BYTES_PER_SVE_VECTOR, |
3337 | VNx16BImode); | |
624d0f07 RS |
3338 | gcc_assert (TYPE_MODE (vectype) == VNx16BImode |
3339 | && TYPE_MODE (vectype) == TYPE_MODE_RAW (vectype) | |
3340 | && TYPE_ALIGN (vectype) == 16 | |
3341 | && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype)), | |
3342 | BYTES_PER_SVE_VECTOR)); | |
683e93d1 | 3343 | num_pr = 1; |
624d0f07 RS |
3344 | } |
3345 | else | |
3346 | { | |
9b070057 RS |
3347 | scalar_mode elmode = SCALAR_TYPE_MODE (eltype); |
3348 | unsigned int elbytes = GET_MODE_SIZE (elmode); | |
624d0f07 | 3349 | poly_uint64 nunits = exact_div (BYTES_PER_SVE_VECTOR, elbytes); |
9b070057 RS |
3350 | machine_mode mode |
3351 | = aarch64_sve_data_mode (elmode, nunits).require (); | |
3352 | vectype = build_vector_type_for_mode (eltype, mode); | |
624d0f07 | 3353 | gcc_assert (VECTOR_MODE_P (TYPE_MODE (vectype)) |
9b070057 RS |
3354 | && TYPE_MODE (vectype) == mode |
3355 | && TYPE_MODE_RAW (vectype) == mode | |
624d0f07 RS |
3356 | && TYPE_ALIGN (vectype) == 128 |
3357 | && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype)), | |
3358 | BITS_PER_SVE_VECTOR)); | |
683e93d1 | 3359 | num_zr = 1; |
624d0f07 RS |
3360 | } |
3361 | vectype = build_distinct_type_copy (vectype); | |
6ff0cdeb | 3362 | gcc_assert (vectype == TYPE_MAIN_VARIANT (vectype)); |
624d0f07 RS |
3363 | SET_TYPE_STRUCTURAL_EQUALITY (vectype); |
3364 | TYPE_ARTIFICIAL (vectype) = 1; | |
f486280c | 3365 | TYPE_INDIVISIBLE_P (vectype) = 1; |
683e93d1 | 3366 | add_sve_type_attribute (vectype, num_zr, num_pr, |
9ded41a3 RS |
3367 | vector_types[i].mangled_name, |
3368 | vector_types[i].acle_name); | |
5002dae3 | 3369 | make_type_sizeless (vectype); |
624d0f07 RS |
3370 | abi_vector_types[i] = vectype; |
3371 | lang_hooks.types.register_builtin_type (vectype, | |
3372 | vector_types[i].abi_name); | |
3373 | } | |
3374 | } | |
3375 | ||
3376 | /* Initialize all compiler built-ins related to SVE that should be | |
3377 | defined at start-up. */ | |
3378 | void | |
3379 | init_builtins () | |
3380 | { | |
3381 | sve_switcher sve; | |
3382 | register_builtin_types (); | |
ab341f50 RS |
3383 | if (in_lto_p) |
3384 | handle_arm_sve_h (); | |
624d0f07 RS |
3385 | } |
3386 | ||
3387 | /* Register vector type TYPE under its arm_sve.h name. */ | |
3388 | static void | |
3389 | register_vector_type (vector_type_index type) | |
3390 | { | |
3391 | tree vectype = abi_vector_types[type]; | |
3392 | tree id = get_identifier (vector_types[type].acle_name); | |
3393 | tree decl = build_decl (input_location, TYPE_DECL, id, vectype); | |
3394 | decl = lang_hooks.decls.pushdecl (decl); | |
3395 | ||
3396 | /* Record the new ACLE type if pushdecl succeeded without error. Use | |
3397 | the ABI type otherwise, so that the type we record at least has the | |
3398 | right form, even if it doesn't have the right name. This should give | |
3399 | better error recovery behavior than installing error_mark_node or | |
3400 | installing an incorrect type. */ | |
ab341f50 RS |
3401 | if (decl |
3402 | && TREE_CODE (decl) == TYPE_DECL | |
624d0f07 RS |
3403 | && TYPE_MAIN_VARIANT (TREE_TYPE (decl)) == vectype) |
3404 | vectype = TREE_TYPE (decl); | |
3405 | acle_vector_types[0][type] = vectype; | |
3406 | } | |
3407 | ||
3408 | /* Register the tuple type that contains NUM_VECTORS vectors of type TYPE. */ | |
3409 | static void | |
3410 | register_tuple_type (unsigned int num_vectors, vector_type_index type) | |
3411 | { | |
3412 | tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE); | |
3413 | ||
9ded41a3 RS |
3414 | /* Work out the structure name. */ |
3415 | char buffer[sizeof ("svbfloat16x4_t")]; | |
3416 | const char *vector_type_name = vector_types[type].acle_name; | |
3417 | snprintf (buffer, sizeof (buffer), "%.*sx%d_t", | |
3418 | (int) strlen (vector_type_name) - 2, vector_type_name, | |
3419 | num_vectors); | |
3420 | ||
624d0f07 RS |
3421 | /* The contents of the type are opaque, so we can define them in any |
3422 | way that maps to the correct ABI type. | |
3423 | ||
3424 | Here we choose to use the same layout as for arm_neon.h, but with | |
3425 | "__val" instead of "val": | |
3426 | ||
3427 | struct svfooxN_t { svfoo_t __val[N]; }; | |
3428 | ||
3429 | (It wouldn't be possible to write that directly in C or C++ for | |
3430 | sizeless types, but that's not a problem for this function.) | |
3431 | ||
3432 | Using arrays simplifies the handling of svget and svset for variable | |
3433 | arguments. */ | |
3434 | tree vector_type = acle_vector_types[0][type]; | |
3435 | tree array_type = build_array_type_nelts (vector_type, num_vectors); | |
3436 | gcc_assert (VECTOR_MODE_P (TYPE_MODE (array_type)) | |
3437 | && TYPE_MODE_RAW (array_type) == TYPE_MODE (array_type) | |
3438 | && TYPE_ALIGN (array_type) == 128); | |
3439 | ||
3440 | tree field = build_decl (input_location, FIELD_DECL, | |
3441 | get_identifier ("__val"), array_type); | |
3442 | DECL_FIELD_CONTEXT (field) = tuple_type; | |
3443 | TYPE_FIELDS (tuple_type) = field; | |
9ded41a3 | 3444 | add_sve_type_attribute (tuple_type, num_vectors, 0, NULL, buffer); |
5002dae3 | 3445 | make_type_sizeless (tuple_type); |
624d0f07 RS |
3446 | layout_type (tuple_type); |
3447 | gcc_assert (VECTOR_MODE_P (TYPE_MODE (tuple_type)) | |
3448 | && TYPE_MODE_RAW (tuple_type) == TYPE_MODE (tuple_type) | |
3449 | && TYPE_ALIGN (tuple_type) == 128); | |
3450 | ||
624d0f07 RS |
3451 | tree decl = build_decl (input_location, TYPE_DECL, |
3452 | get_identifier (buffer), tuple_type); | |
3453 | TYPE_NAME (tuple_type) = decl; | |
3454 | TYPE_STUB_DECL (tuple_type) = decl; | |
3455 | lang_hooks.decls.pushdecl (decl); | |
3456 | /* ??? Undo the effect of set_underlying_type for C. The C frontend | |
3457 | doesn't recognize DECL as a built-in because (as intended) the decl has | |
3458 | a real location instead of BUILTINS_LOCATION. The frontend therefore | |
3459 | treats the decl like a normal C "typedef struct foo foo;", expecting | |
3460 | the type for tag "struct foo" to have a dummy unnamed TYPE_DECL instead | |
3461 | of the named one we attached above. It then sets DECL_ORIGINAL_TYPE | |
3462 | on the supposedly unnamed decl, creating a circularity that upsets | |
3463 | dwarf2out. | |
3464 | ||
3465 | We don't want to follow the normal C model and create "struct foo" | |
3466 | tags for tuple types since (a) the types are supposed to be opaque | |
3467 | and (b) they couldn't be defined as a real struct anyway. Treating | |
3468 | the TYPE_DECLs as "typedef struct foo foo;" without creating | |
3469 | "struct foo" would lead to confusing error messages. */ | |
3470 | DECL_ORIGINAL_TYPE (decl) = NULL_TREE; | |
3471 | ||
3472 | acle_vector_types[num_vectors - 1][type] = tuple_type; | |
3473 | } | |
3474 | ||
3475 | /* Register the svpattern enum. */ | |
3476 | static void | |
3477 | register_svpattern () | |
3478 | { | |
3479 | auto_vec<string_int_pair, 32> values; | |
3480 | #define PUSH(UPPER, LOWER, VALUE) \ | |
3481 | values.quick_push (string_int_pair ("SV_" #UPPER, VALUE)); | |
3482 | AARCH64_FOR_SVPATTERN (PUSH) | |
3483 | #undef PUSH | |
3484 | ||
3485 | acle_svpattern = lang_hooks.types.simulate_enum_decl (input_location, | |
3486 | "svpattern", values); | |
3487 | } | |
3488 | ||
3489 | /* Register the svprfop enum. */ | |
3490 | static void | |
3491 | register_svprfop () | |
3492 | { | |
3493 | auto_vec<string_int_pair, 16> values; | |
3494 | #define PUSH(UPPER, LOWER, VALUE) \ | |
3495 | values.quick_push (string_int_pair ("SV_" #UPPER, VALUE)); | |
3496 | AARCH64_FOR_SVPRFOP (PUSH) | |
3497 | #undef PUSH | |
3498 | ||
3499 | acle_svprfop = lang_hooks.types.simulate_enum_decl (input_location, | |
3500 | "svprfop", values); | |
3501 | } | |
3502 | ||
3503 | /* Implement #pragma GCC aarch64 "arm_sve.h". */ | |
3504 | void | |
3505 | handle_arm_sve_h () | |
3506 | { | |
3507 | if (function_table) | |
3508 | { | |
3509 | error ("duplicate definition of %qs", "arm_sve.h"); | |
3510 | return; | |
3511 | } | |
3512 | ||
3513 | sve_switcher sve; | |
3514 | ||
3515 | /* Define the vector and tuple types. */ | |
3516 | for (unsigned int type_i = 0; type_i < NUM_VECTOR_TYPES; ++type_i) | |
3517 | { | |
3518 | vector_type_index type = vector_type_index (type_i); | |
3519 | register_vector_type (type); | |
3520 | if (type != VECTOR_TYPE_svbool_t) | |
3521 | for (unsigned int count = 2; count <= MAX_TUPLE_SIZE; ++count) | |
3522 | register_tuple_type (count, type); | |
3523 | } | |
3524 | ||
3525 | /* Define the enums. */ | |
3526 | register_svpattern (); | |
3527 | register_svprfop (); | |
3528 | ||
3529 | /* Define the functions. */ | |
3530 | function_table = new hash_table<registered_function_hasher> (1023); | |
3531 | function_builder builder; | |
3532 | for (unsigned int i = 0; i < ARRAY_SIZE (function_groups); ++i) | |
3533 | builder.register_function_group (function_groups[i]); | |
3534 | } | |
3535 | ||
3536 | /* Return the function decl with SVE function subcode CODE, or error_mark_node | |
3537 | if no such function exists. */ | |
3538 | tree | |
3539 | builtin_decl (unsigned int code, bool) | |
3540 | { | |
3541 | if (code >= vec_safe_length (registered_functions)) | |
3542 | return error_mark_node; | |
3543 | return (*registered_functions)[code]->decl; | |
3544 | } | |
3545 | ||
3546 | /* If we're implementing manual overloading, check whether the SVE | |
3547 | function with subcode CODE is overloaded, and if so attempt to | |
3548 | determine the corresponding non-overloaded function. The call | |
3549 | occurs at location LOCATION and has the arguments given by ARGLIST. | |
3550 | ||
3551 | If the call is erroneous, report an appropriate error and return | |
3552 | error_mark_node. Otherwise, if the function is overloaded, return | |
3553 | the decl of the non-overloaded function. Return NULL_TREE otherwise, | |
3554 | indicating that the call should be processed in the normal way. */ | |
3555 | tree | |
3556 | resolve_overloaded_builtin (location_t location, unsigned int code, | |
3557 | vec<tree, va_gc> *arglist) | |
3558 | { | |
3559 | if (code >= vec_safe_length (registered_functions)) | |
3560 | return NULL_TREE; | |
3561 | ||
3562 | registered_function &rfn = *(*registered_functions)[code]; | |
3563 | if (rfn.overloaded_p) | |
3564 | return function_resolver (location, rfn.instance, rfn.decl, | |
3565 | *arglist).resolve (); | |
3566 | return NULL_TREE; | |
3567 | } | |
3568 | ||
3569 | /* Perform any semantic checks needed for a call to the SVE function | |
3570 | with subcode CODE, such as testing for integer constant expressions. | |
3571 | The call occurs at location LOCATION and has NARGS arguments, | |
3572 | given by ARGS. FNDECL is the original function decl, before | |
3573 | overload resolution. | |
3574 | ||
3575 | Return true if the call is valid, otherwise report a suitable error. */ | |
3576 | bool | |
3577 | check_builtin_call (location_t location, vec<location_t>, unsigned int code, | |
3578 | tree fndecl, unsigned int nargs, tree *args) | |
3579 | { | |
3580 | const registered_function &rfn = *(*registered_functions)[code]; | |
3581 | if (!check_required_extensions (location, rfn.decl, rfn.required_extensions)) | |
3582 | return false; | |
3583 | return function_checker (location, rfn.instance, fndecl, | |
3584 | TREE_TYPE (rfn.decl), nargs, args).check (); | |
3585 | } | |
3586 | ||
3587 | /* Attempt to fold STMT, given that it's a call to the SVE function | |
3588 | with subcode CODE. Return the new statement on success and null | |
3589 | on failure. Insert any other new statements at GSI. */ | |
3590 | gimple * | |
3591 | gimple_fold_builtin (unsigned int code, gimple_stmt_iterator *gsi, gcall *stmt) | |
3592 | { | |
3593 | registered_function &rfn = *(*registered_functions)[code]; | |
3594 | return gimple_folder (rfn.instance, rfn.decl, gsi, stmt).fold (); | |
3595 | } | |
3596 | ||
3597 | /* Expand a call to the SVE function with subcode CODE. EXP is the call | |
3598 | expression and TARGET is the preferred location for the result. | |
3599 | Return the value of the lhs. */ | |
3600 | rtx | |
3601 | expand_builtin (unsigned int code, tree exp, rtx target) | |
3602 | { | |
3603 | registered_function &rfn = *(*registered_functions)[code]; | |
3604 | if (!check_required_extensions (EXPR_LOCATION (exp), rfn.decl, | |
3605 | rfn.required_extensions)) | |
3606 | return target; | |
3607 | return function_expander (rfn.instance, rfn.decl, exp, target).expand (); | |
3608 | } | |
3609 | ||
624d0f07 RS |
3610 | /* If TYPE is a built-in type defined by the SVE ABI, return the mangled name, |
3611 | otherwise return NULL. */ | |
3612 | const char * | |
3613 | mangle_builtin_type (const_tree type) | |
3614 | { | |
683e93d1 RS |
3615 | /* ??? The C++ frontend normally strips qualifiers and attributes before |
3616 | calling this hook, adding separate mangling for attributes that affect | |
3617 | type identity. Fortunately the type copy will have the same TYPE_NAME | |
3618 | as the original, so we can get the attributes from there. */ | |
3619 | if (TYPE_NAME (type) && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL) | |
3620 | type = TREE_TYPE (TYPE_NAME (type)); | |
3621 | if (tree attr = lookup_sve_type_attribute (type)) | |
3622 | if (tree id = TREE_VALUE (chain_index (2, TREE_VALUE (attr)))) | |
3623 | return IDENTIFIER_POINTER (id); | |
624d0f07 RS |
3624 | return NULL; |
3625 | } | |
3626 | ||
683e93d1 RS |
3627 | /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. */ |
3628 | bool | |
3629 | builtin_type_p (const_tree type) | |
624d0f07 | 3630 | { |
683e93d1 | 3631 | return lookup_sve_type_attribute (type); |
624d0f07 RS |
3632 | } |
3633 | ||
683e93d1 RS |
3634 | /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. |
3635 | If so, store the number of constituent SVE vectors in *NUM_ZR and the | |
3636 | number of constituent SVE predicates in *NUM_PR. */ | |
624d0f07 | 3637 | bool |
683e93d1 | 3638 | builtin_type_p (const_tree type, unsigned int *num_zr, unsigned int *num_pr) |
624d0f07 | 3639 | { |
683e93d1 RS |
3640 | if (tree attr = lookup_sve_type_attribute (type)) |
3641 | { | |
3642 | tree num_zr_node = TREE_VALUE (attr); | |
3643 | tree num_pr_node = TREE_CHAIN (num_zr_node); | |
3644 | *num_zr = tree_to_uhwi (TREE_VALUE (num_zr_node)); | |
3645 | *num_pr = tree_to_uhwi (TREE_VALUE (num_pr_node)); | |
3646 | return true; | |
3647 | } | |
3648 | return false; | |
624d0f07 RS |
3649 | } |
3650 | ||
9ded41a3 RS |
3651 | /* ATTRS is the attribute list for a sizeless SVE type. Return the |
3652 | attributes of the associated fixed-length SVE type, taking the | |
3653 | "SVE type" attributes from NEW_SVE_TYPE_ARGS. */ | |
3654 | static tree | |
3655 | get_arm_sve_vector_bits_attributes (tree old_attrs, tree new_sve_type_args) | |
3656 | { | |
3657 | tree new_attrs = NULL_TREE; | |
3658 | tree *ptr = &new_attrs; | |
3659 | for (tree attr = old_attrs; attr; attr = TREE_CHAIN (attr)) | |
3660 | { | |
3661 | tree name = get_attribute_name (attr); | |
3662 | if (is_attribute_p ("SVE sizeless type", name)) | |
3663 | continue; | |
3664 | ||
3665 | tree args = TREE_VALUE (attr); | |
3666 | if (is_attribute_p ("SVE type", name)) | |
3667 | args = new_sve_type_args; | |
3668 | *ptr = tree_cons (TREE_PURPOSE (attr), args, NULL_TREE); | |
3669 | ptr = &TREE_CHAIN (*ptr); | |
3670 | } | |
3671 | return new_attrs; | |
3672 | } | |
3673 | ||
38e62001 RS |
3674 | /* An attribute callback for the "arm_sve_vector_bits" attribute. */ |
3675 | tree | |
3676 | handle_arm_sve_vector_bits_attribute (tree *node, tree, tree args, int, | |
3677 | bool *no_add_attrs) | |
3678 | { | |
3679 | *no_add_attrs = true; | |
3680 | ||
3681 | tree type = *node; | |
9ded41a3 RS |
3682 | tree attr = lookup_sve_type_attribute (type); |
3683 | if (!attr) | |
38e62001 RS |
3684 | { |
3685 | error ("%qs applied to non-SVE type %qT", "arm_sve_vector_bits", type); | |
3686 | return NULL_TREE; | |
3687 | } | |
3688 | ||
9ded41a3 RS |
3689 | if (!VECTOR_TYPE_P (type)) |
3690 | { | |
3691 | error ("%qs applied to non-vector type %qT", | |
3692 | "arm_sve_vector_bits", type); | |
3693 | return NULL_TREE; | |
3694 | } | |
3695 | ||
3696 | if (!sizeless_type_p (type)) | |
3697 | { | |
3698 | error ("%qs applied to type %qT, which already has a size", | |
3699 | "arm_sve_vector_bits", type); | |
3700 | return NULL_TREE; | |
3701 | } | |
3702 | ||
38e62001 RS |
3703 | tree size = TREE_VALUE (args); |
3704 | if (TREE_CODE (size) != INTEGER_CST) | |
3705 | { | |
3706 | error ("%qs requires an integer constant expression", | |
3707 | "arm_sve_vector_bits"); | |
3708 | return NULL_TREE; | |
3709 | } | |
3710 | ||
3711 | unsigned HOST_WIDE_INT value = tree_to_uhwi (size); | |
3712 | if (maybe_ne (value, BITS_PER_SVE_VECTOR)) | |
3713 | { | |
3714 | warning (OPT_Wattributes, "unsupported SVE vector size"); | |
3715 | return NULL_TREE; | |
3716 | } | |
3717 | ||
9ded41a3 RS |
3718 | /* Construct a new list of "SVE type" attribute arguments. */ |
3719 | tree new_sve_type_args = copy_list (TREE_VALUE (attr)); | |
3720 | ||
3721 | /* Mangle the type as an instance of the imaginary template: | |
3722 | ||
3723 | __SVE_VLS<typename, unsigned> | |
3724 | ||
3725 | where the first parameter is the SVE type and where the second | |
3726 | parameter is the SVE vector length in bits. */ | |
3727 | tree mangled_name_node = chain_index (2, new_sve_type_args); | |
3728 | const char *old_mangled_name | |
3729 | = IDENTIFIER_POINTER (TREE_VALUE (mangled_name_node)); | |
3730 | char *new_mangled_name | |
3731 | = xasprintf ("9__SVE_VLSI%sLj%dEE", old_mangled_name, (int) value); | |
3732 | TREE_VALUE (mangled_name_node) = get_identifier (new_mangled_name); | |
3733 | free (new_mangled_name); | |
3734 | ||
38e62001 RS |
3735 | /* FIXME: The type ought to be a distinct copy in all cases, but |
3736 | currently that makes the C frontend reject conversions between | |
3737 | svbool_t and its fixed-length variants. Using a type variant | |
3738 | avoids that but means that we treat some ambiguous combinations | |
3739 | as valid. */ | |
2c814af6 RS |
3740 | tree new_type; |
3741 | tree base_type = TYPE_MAIN_VARIANT (type); | |
38e62001 | 3742 | if (lang_GNU_C () && VECTOR_BOOLEAN_TYPE_P (type)) |
2c814af6 | 3743 | new_type = build_variant_type_copy (base_type); |
38e62001 | 3744 | else |
2c814af6 RS |
3745 | new_type = build_distinct_type_copy (base_type); |
3746 | ||
9ded41a3 RS |
3747 | /* Construct a TYPE_DECL for the new type. This serves two purposes: |
3748 | ||
3749 | - It ensures we don't print the original TYPE_DECL in error messages. | |
3750 | Printing the original name would be confusing because there are | |
3751 | situations in which the distinction between the original type and | |
3752 | the new type matters. For example: | |
3753 | ||
3754 | __SVInt8_t __attribute__((arm_sve_vector_bits(512))) *a; | |
3755 | __SVInt8_t *b; | |
3756 | ||
3757 | a = b; | |
3758 | ||
3759 | is invalid in C++, but without this, we'd print both types in | |
3760 | the same way. | |
3761 | ||
3762 | - Having a separate TYPE_DECL is necessary to ensure that C++ | |
3763 | mangling works correctly. See mangle_builtin_type for details. | |
3764 | ||
3765 | The name of the decl is something like: | |
3766 | ||
3767 | svint8_t __attribute__((arm_sve_vector_bits(512))) | |
3768 | ||
3769 | This is a compromise. It would be more accurate to use something like: | |
3770 | ||
3771 | __SVInt8_t __attribute__((arm_sve_vector_bits(512))) | |
3772 | ||
3773 | but the <arm_sve.h> name is likely to be more meaningful. */ | |
3774 | tree acle_name_node = TREE_CHAIN (mangled_name_node); | |
3775 | const char *old_type_name = IDENTIFIER_POINTER (TREE_VALUE (acle_name_node)); | |
3776 | char *new_type_name | |
3777 | = xasprintf ("%s __attribute__((arm_sve_vector_bits(%d)))", | |
3778 | old_type_name, (int) value); | |
3779 | tree decl = build_decl (BUILTINS_LOCATION, TYPE_DECL, | |
3780 | get_identifier (new_type_name), new_type); | |
3781 | DECL_ARTIFICIAL (decl) = 1; | |
3782 | TYPE_NAME (new_type) = decl; | |
3783 | free (new_type_name); | |
3784 | ||
2c814af6 RS |
3785 | /* Allow the GNU vector extensions to be applied to vectors. |
3786 | The extensions aren't yet defined for packed predicates, | |
3787 | so continue to treat them as abstract entities for now. */ | |
3788 | if (!VECTOR_BOOLEAN_TYPE_P (new_type)) | |
3789 | TYPE_INDIVISIBLE_P (new_type) = 0; | |
38e62001 RS |
3790 | |
3791 | /* The new type is a normal sized type; it doesn't have the same | |
3792 | restrictions as sizeless types. */ | |
2c814af6 | 3793 | TYPE_ATTRIBUTES (new_type) |
9ded41a3 RS |
3794 | = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (new_type), |
3795 | new_sve_type_args); | |
38e62001 | 3796 | |
2c814af6 RS |
3797 | /* Apply the relevant attributes, qualifiers and alignment of TYPE, |
3798 | if they differ from the original (sizeless) BASE_TYPE. */ | |
3799 | if (TYPE_ATTRIBUTES (base_type) != TYPE_ATTRIBUTES (type) | |
3800 | || TYPE_QUALS (base_type) != TYPE_QUALS (type)) | |
3801 | { | |
9ded41a3 RS |
3802 | tree attrs |
3803 | = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (type), | |
3804 | new_sve_type_args); | |
2c814af6 RS |
3805 | new_type = build_type_attribute_qual_variant (new_type, attrs, |
3806 | TYPE_QUALS (type)); | |
3807 | } | |
3808 | if (TYPE_ALIGN (base_type) != TYPE_ALIGN (type)) | |
3809 | new_type = build_aligned_type (new_type, TYPE_ALIGN (type)); | |
38e62001 | 3810 | |
2c814af6 | 3811 | *node = new_type; |
38e62001 RS |
3812 | return NULL_TREE; |
3813 | } | |
3814 | ||
65ef05d0 RS |
3815 | /* Implement TARGET_VERIFY_TYPE_CONTEXT for SVE types. */ |
3816 | bool | |
3817 | verify_type_context (location_t loc, type_context_kind context, | |
3818 | const_tree type, bool silent_p) | |
3819 | { | |
5002dae3 | 3820 | if (!sizeless_type_p (type)) |
65ef05d0 RS |
3821 | return true; |
3822 | ||
3823 | switch (context) | |
3824 | { | |
3825 | case TCTX_SIZEOF: | |
3826 | case TCTX_STATIC_STORAGE: | |
3827 | if (!silent_p) | |
3828 | error_at (loc, "SVE type %qT does not have a fixed size", type); | |
3829 | return false; | |
3830 | ||
3831 | case TCTX_ALIGNOF: | |
3832 | if (!silent_p) | |
3833 | error_at (loc, "SVE type %qT does not have a defined alignment", type); | |
3834 | return false; | |
3835 | ||
3836 | case TCTX_THREAD_STORAGE: | |
3837 | if (!silent_p) | |
3838 | error_at (loc, "variables of type %qT cannot have thread-local" | |
3839 | " storage duration", type); | |
3840 | return false; | |
3841 | ||
3842 | case TCTX_POINTER_ARITH: | |
3843 | if (!silent_p) | |
3844 | error_at (loc, "arithmetic on pointer to SVE type %qT", type); | |
3845 | return false; | |
3846 | ||
3847 | case TCTX_FIELD: | |
3848 | if (silent_p) | |
3849 | ; | |
3850 | else if (lang_GNU_CXX ()) | |
3851 | error_at (loc, "member variables cannot have SVE type %qT", type); | |
3852 | else | |
3853 | error_at (loc, "fields cannot have SVE type %qT", type); | |
3854 | return false; | |
3855 | ||
3856 | case TCTX_ARRAY_ELEMENT: | |
3857 | if (!silent_p) | |
3858 | error_at (loc, "array elements cannot have SVE type %qT", type); | |
3859 | return false; | |
02a32ab4 RS |
3860 | |
3861 | case TCTX_ALLOCATION: | |
3862 | if (!silent_p) | |
3863 | error_at (loc, "cannot allocate objects with SVE type %qT", type); | |
3864 | return false; | |
3865 | ||
3866 | case TCTX_DEALLOCATION: | |
3867 | if (!silent_p) | |
3868 | error_at (loc, "cannot delete objects with SVE type %qT", type); | |
3869 | return false; | |
3870 | ||
3871 | case TCTX_EXCEPTIONS: | |
3872 | if (!silent_p) | |
3873 | error_at (loc, "cannot throw or catch SVE type %qT", type); | |
3874 | return false; | |
3875 | ||
3876 | case TCTX_CAPTURE_BY_COPY: | |
3877 | if (!silent_p) | |
3878 | error_at (loc, "capture by copy of SVE type %qT", type); | |
3879 | return false; | |
65ef05d0 RS |
3880 | } |
3881 | gcc_unreachable (); | |
3882 | } | |
3883 | ||
624d0f07 RS |
3884 | } |
3885 | ||
3886 | using namespace aarch64_sve; | |
3887 | ||
3888 | inline void | |
3889 | gt_ggc_mx (function_instance *) | |
3890 | { | |
3891 | } | |
3892 | ||
3893 | inline void | |
3894 | gt_pch_nx (function_instance *) | |
3895 | { | |
3896 | } | |
3897 | ||
3898 | inline void | |
3899 | gt_pch_nx (function_instance *, void (*) (void *, void *), void *) | |
3900 | { | |
3901 | } | |
3902 | ||
3903 | #include "gt-aarch64-sve-builtins.h" |