1 /* ACLE support for AArch64 SVE
2 Copyright (C) 2018-2022 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
11 GCC is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
20 #define IN_TARGET_CODE 1
24 #include "coretypes.h"
30 #include "insn-codes.h"
33 #include "diagnostic.h"
35 #include "basic-block.h"
37 #include "fold-const.h"
39 #include "gimple-iterator.h"
43 #include "tree-vector-builder.h"
44 #include "stor-layout.h"
47 #include "gimple-fold.h"
48 #include "langhooks.h"
49 #include "stringpool.h"
51 #include "aarch64-sve-builtins.h"
52 #include "aarch64-sve-builtins-base.h"
53 #include "aarch64-sve-builtins-sve2.h"
54 #include "aarch64-sve-builtins-shapes.h"
56 namespace aarch64_sve
{
58 /* Static information about each single-predicate or single-vector
60 struct vector_type_info
62 /* The name of the type as declared by arm_sve.h. */
63 const char *acle_name
;
65 /* The name of the type specified in AAPCS64. The type is always
66 available under this name, even when arm_sve.h isn't included. */
69 /* The C++ mangling of ABI_NAME. */
70 const char *mangled_name
;
73 /* Describes a function decl. */
74 class GTY(()) registered_function
77 /* The ACLE function that the decl represents. */
78 function_instance instance
GTY ((skip
));
80 /* The decl itself. */
83 /* The architecture extensions that the function requires, as a set of
84 AARCH64_FL_* flags. */
85 aarch64_feature_flags required_extensions
;
87 /* True if the decl represents an overloaded function that needs to be
88 resolved by function_resolver. */
92 /* Hash traits for registered_function. */
93 struct registered_function_hasher
: nofree_ptr_hash
<registered_function
>
95 typedef function_instance compare_type
;
97 static hashval_t
hash (value_type
);
98 static bool equal (value_type
, const compare_type
&);
101 /* Information about each single-predicate or single-vector type. */
102 static CONSTEXPR
const vector_type_info vector_types
[] = {
103 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
104 { #ACLE_NAME, #ABI_NAME, "u" #NCHARS #ABI_NAME },
105 #include "aarch64-sve-builtins.def"
108 /* The function name suffix associated with each predication type. */
109 static const char *const pred_suffixes
[NUM_PREDS
+ 1] = {
118 /* Static information about each mode_suffix_index. */
119 CONSTEXPR
const mode_suffix_info mode_suffixes
[] = {
120 #define VECTOR_TYPE_none NUM_VECTOR_TYPES
121 #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) \
122 { "_" #NAME, VECTOR_TYPE_##BASE, VECTOR_TYPE_##DISPLACEMENT, UNITS_##UNITS },
123 #include "aarch64-sve-builtins.def"
124 #undef VECTOR_TYPE_none
125 { "", NUM_VECTOR_TYPES
, NUM_VECTOR_TYPES
, UNITS_none
}
128 /* Static information about each type_suffix_index. */
129 CONSTEXPR
const type_suffix_info type_suffixes
[NUM_TYPE_SUFFIXES
+ 1] = {
130 #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \
132 VECTOR_TYPE_##ACLE_TYPE, \
135 BITS / BITS_PER_UNIT, \
136 TYPE_##CLASS == TYPE_signed || TYPE_##CLASS == TYPE_unsigned, \
137 TYPE_##CLASS == TYPE_unsigned, \
138 TYPE_##CLASS == TYPE_float, \
139 TYPE_##CLASS == TYPE_bool, \
142 #include "aarch64-sve-builtins.def"
143 { "", NUM_VECTOR_TYPES
, TYPE_bool
, 0, 0, false, false, false, false,
147 /* Define a TYPES_<combination> macro for each combination of type
148 suffixes that an ACLE function can have, where <combination> is the
149 name used in DEF_SVE_FUNCTION entries.
151 Use S (T) for single type suffix T and D (T1, T2) for a pair of type
152 suffixes T1 and T2. Use commas to separate the suffixes.
154 Although the order shouldn't matter, the convention is to sort the
155 suffixes lexicographically after dividing suffixes into a type
156 class ("b", "f", etc.) and a numerical bit count. */
158 /* _b8 _b16 _b32 _b64. */
159 #define TYPES_all_pred(S, D) \
160 S (b8), S (b16), S (b32), S (b64)
162 /* _f16 _f32 _f64. */
163 #define TYPES_all_float(S, D) \
164 S (f16), S (f32), S (f64)
166 /* _s8 _s16 _s32 _s64. */
167 #define TYPES_all_signed(S, D) \
168 S (s8), S (s16), S (s32), S (s64)
171 _s8 _s16 _s32 _s64. */
172 #define TYPES_all_float_and_signed(S, D) \
173 TYPES_all_float (S, D), TYPES_all_signed (S, D)
175 /* _u8 _u16 _u32 _u64. */
176 #define TYPES_all_unsigned(S, D) \
177 S (u8), S (u16), S (u32), S (u64)
179 /* _s8 _s16 _s32 _s64
180 _u8 _u16 _u32 _u64. */
181 #define TYPES_all_integer(S, D) \
182 TYPES_all_signed (S, D), TYPES_all_unsigned (S, D)
186 _u8 _u16 _u32 _u64. */
187 #define TYPES_all_arith(S, D) \
188 TYPES_all_float (S, D), TYPES_all_integer (S, D)
193 _u8 _u16 _u32 _u64. */
194 #define TYPES_all_data(S, D) \
195 S (bf16), TYPES_all_arith (S, D)
198 #define TYPES_b(S, D) \
202 #define TYPES_b_unsigned(S, D) \
207 #define TYPES_b_integer(S, D) \
208 S (s8), TYPES_b_unsigned (S, D)
212 #define TYPES_bh_integer(S, D) \
213 S (s8), S (s16), S (u8), S (u16)
216 #define TYPES_bs_unsigned(S, D) \
220 #define TYPES_bhs_signed(S, D) \
221 S (s8), S (s16), S (s32)
224 #define TYPES_bhs_unsigned(S, D) \
225 S (u8), S (u16), S (u32)
229 #define TYPES_bhs_integer(S, D) \
230 TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D)
234 #define TYPES_h_integer(S, D) \
238 #define TYPES_hs_signed(S, D) \
243 #define TYPES_hs_integer(S, D) \
244 TYPES_hs_signed (S, D), S (u16), S (u32)
247 #define TYPES_hs_float(S, D) \
251 #define TYPES_hd_unsigned(S, D) \
254 /* _s16 _s32 _s64. */
255 #define TYPES_hsd_signed(S, D) \
256 S (s16), S (s32), S (s64)
260 #define TYPES_hsd_integer(S, D) \
261 TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64)
264 #define TYPES_s_float(S, D) \
270 #define TYPES_s_float_hsd_integer(S, D) \
271 TYPES_s_float (S, D), TYPES_hsd_integer (S, D)
276 #define TYPES_s_float_sd_integer(S, D) \
277 TYPES_s_float (S, D), TYPES_sd_integer (S, D)
280 #define TYPES_s_signed(S, D) \
284 #define TYPES_s_unsigned(S, D) \
288 #define TYPES_s_integer(S, D) \
289 TYPES_s_signed (S, D), TYPES_s_unsigned (S, D)
292 #define TYPES_sd_signed(S, D) \
296 #define TYPES_sd_unsigned(S, D) \
301 #define TYPES_sd_integer(S, D) \
302 TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D)
307 #define TYPES_sd_data(S, D) \
308 S (f32), S (f64), TYPES_sd_integer (S, D)
313 #define TYPES_all_float_and_sd_integer(S, D) \
314 TYPES_all_float (S, D), TYPES_sd_integer (S, D)
317 #define TYPES_d_float(S, D) \
321 #define TYPES_d_unsigned(S, D) \
326 #define TYPES_d_integer(S, D) \
327 S (s64), TYPES_d_unsigned (S, D)
332 #define TYPES_d_data(S, D) \
333 TYPES_d_float (S, D), TYPES_d_integer (S, D)
335 /* All the type combinations allowed by svcvt. */
336 #define TYPES_cvt(S, D) \
337 D (f16, f32), D (f16, f64), \
338 D (f16, s16), D (f16, s32), D (f16, s64), \
339 D (f16, u16), D (f16, u32), D (f16, u64), \
341 D (f32, f16), D (f32, f64), \
342 D (f32, s32), D (f32, s64), \
343 D (f32, u32), D (f32, u64), \
345 D (f64, f16), D (f64, f32), \
346 D (f64, s32), D (f64, s64), \
347 D (f64, u32), D (f64, u64), \
350 D (s32, f16), D (s32, f32), D (s32, f64), \
351 D (s64, f16), D (s64, f32), D (s64, f64), \
354 D (u32, f16), D (u32, f32), D (u32, f64), \
355 D (u64, f16), D (u64, f32), D (u64, f64)
358 #define TYPES_cvt_bfloat(S, D) \
363 #define TYPES_cvt_long(S, D) \
364 D (f32, f16), D (f64, f32)
367 #define TYPES_cvt_narrow_s(S, D) \
372 #define TYPES_cvt_narrow(S, D) \
373 D (f16, f32), TYPES_cvt_narrow_s (S, D)
375 /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 }
377 #define TYPES_inc_dec_n1(D, A) \
378 D (A, b8), D (A, b16), D (A, b32), D (A, b64)
379 #define TYPES_inc_dec_n(S, D) \
380 TYPES_inc_dec_n1 (D, s32), \
381 TYPES_inc_dec_n1 (D, s64), \
382 TYPES_inc_dec_n1 (D, u32), \
383 TYPES_inc_dec_n1 (D, u64)
385 /* { _bf16 } { _bf16 }
386 { _f16 _f32 _f64 } { _f16 _f32 _f64 }
387 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 }
388 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }. */
389 #define TYPES_reinterpret1(D, A) \
391 D (A, f16), D (A, f32), D (A, f64), \
392 D (A, s8), D (A, s16), D (A, s32), D (A, s64), \
393 D (A, u8), D (A, u16), D (A, u32), D (A, u64)
394 #define TYPES_reinterpret(S, D) \
395 TYPES_reinterpret1 (D, bf16), \
396 TYPES_reinterpret1 (D, f16), \
397 TYPES_reinterpret1 (D, f32), \
398 TYPES_reinterpret1 (D, f64), \
399 TYPES_reinterpret1 (D, s8), \
400 TYPES_reinterpret1 (D, s16), \
401 TYPES_reinterpret1 (D, s32), \
402 TYPES_reinterpret1 (D, s64), \
403 TYPES_reinterpret1 (D, u8), \
404 TYPES_reinterpret1 (D, u16), \
405 TYPES_reinterpret1 (D, u32), \
406 TYPES_reinterpret1 (D, u64)
408 /* { _b8 _b16 _b32 _b64 } x { _s32 _s64 }
410 #define TYPES_while1(D, bn) \
411 D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64)
412 #define TYPES_while(S, D) \
413 TYPES_while1 (D, b8), \
414 TYPES_while1 (D, b16), \
415 TYPES_while1 (D, b32), \
416 TYPES_while1 (D, b64)
418 /* Describe a pair of type suffixes in which only the first is used. */
419 #define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES }
421 /* Describe a pair of type suffixes in which both are used. */
422 #define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y }
424 /* Create an array that can be used in aarch64-sve-builtins.def to
425 select the type suffixes in TYPES_<NAME>. */
426 #define DEF_SVE_TYPES_ARRAY(NAME) \
427 static const type_suffix_pair types_##NAME[] = { \
428 TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \
429 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \
432 /* For functions that don't take any type suffixes. */
433 static const type_suffix_pair types_none
[] = {
434 { NUM_TYPE_SUFFIXES
, NUM_TYPE_SUFFIXES
},
435 { NUM_TYPE_SUFFIXES
, NUM_TYPE_SUFFIXES
}
438 /* Create an array for each TYPES_<combination> macro above. */
439 DEF_SVE_TYPES_ARRAY (all_pred
);
440 DEF_SVE_TYPES_ARRAY (all_float
);
441 DEF_SVE_TYPES_ARRAY (all_signed
);
442 DEF_SVE_TYPES_ARRAY (all_float_and_signed
);
443 DEF_SVE_TYPES_ARRAY (all_unsigned
);
444 DEF_SVE_TYPES_ARRAY (all_integer
);
445 DEF_SVE_TYPES_ARRAY (all_arith
);
446 DEF_SVE_TYPES_ARRAY (all_data
);
447 DEF_SVE_TYPES_ARRAY (b
);
448 DEF_SVE_TYPES_ARRAY (b_unsigned
);
449 DEF_SVE_TYPES_ARRAY (b_integer
);
450 DEF_SVE_TYPES_ARRAY (bh_integer
);
451 DEF_SVE_TYPES_ARRAY (bs_unsigned
);
452 DEF_SVE_TYPES_ARRAY (bhs_signed
);
453 DEF_SVE_TYPES_ARRAY (bhs_unsigned
);
454 DEF_SVE_TYPES_ARRAY (bhs_integer
);
455 DEF_SVE_TYPES_ARRAY (h_integer
);
456 DEF_SVE_TYPES_ARRAY (hs_signed
);
457 DEF_SVE_TYPES_ARRAY (hs_integer
);
458 DEF_SVE_TYPES_ARRAY (hs_float
);
459 DEF_SVE_TYPES_ARRAY (hd_unsigned
);
460 DEF_SVE_TYPES_ARRAY (hsd_signed
);
461 DEF_SVE_TYPES_ARRAY (hsd_integer
);
462 DEF_SVE_TYPES_ARRAY (s_float
);
463 DEF_SVE_TYPES_ARRAY (s_float_hsd_integer
);
464 DEF_SVE_TYPES_ARRAY (s_float_sd_integer
);
465 DEF_SVE_TYPES_ARRAY (s_signed
);
466 DEF_SVE_TYPES_ARRAY (s_unsigned
);
467 DEF_SVE_TYPES_ARRAY (s_integer
);
468 DEF_SVE_TYPES_ARRAY (sd_signed
);
469 DEF_SVE_TYPES_ARRAY (sd_unsigned
);
470 DEF_SVE_TYPES_ARRAY (sd_integer
);
471 DEF_SVE_TYPES_ARRAY (sd_data
);
472 DEF_SVE_TYPES_ARRAY (all_float_and_sd_integer
);
473 DEF_SVE_TYPES_ARRAY (d_float
);
474 DEF_SVE_TYPES_ARRAY (d_unsigned
);
475 DEF_SVE_TYPES_ARRAY (d_integer
);
476 DEF_SVE_TYPES_ARRAY (d_data
);
477 DEF_SVE_TYPES_ARRAY (cvt
);
478 DEF_SVE_TYPES_ARRAY (cvt_bfloat
);
479 DEF_SVE_TYPES_ARRAY (cvt_long
);
480 DEF_SVE_TYPES_ARRAY (cvt_narrow_s
);
481 DEF_SVE_TYPES_ARRAY (cvt_narrow
);
482 DEF_SVE_TYPES_ARRAY (inc_dec_n
);
483 DEF_SVE_TYPES_ARRAY (reinterpret
);
484 DEF_SVE_TYPES_ARRAY (while);
486 /* Used by functions that have no governing predicate. */
487 static const predication_index preds_none
[] = { PRED_none
, NUM_PREDS
};
489 /* Used by functions that have a governing predicate but do not have an
491 static const predication_index preds_implicit
[] = { PRED_implicit
, NUM_PREDS
};
493 /* Used by functions that allow merging and "don't care" predication,
494 but are not suitable for predicated MOVPRFX. */
495 static const predication_index preds_mx
[] = {
496 PRED_m
, PRED_x
, NUM_PREDS
499 /* Used by functions that allow merging, zeroing and "don't care"
501 static const predication_index preds_mxz
[] = {
502 PRED_m
, PRED_x
, PRED_z
, NUM_PREDS
505 /* Used by functions that have the mxz predicated forms above, and in addition
506 have an unpredicated form. */
507 static const predication_index preds_mxz_or_none
[] = {
508 PRED_m
, PRED_x
, PRED_z
, PRED_none
, NUM_PREDS
511 /* Used by functions that allow merging and zeroing predication but have
513 static const predication_index preds_mz
[] = { PRED_m
, PRED_z
, NUM_PREDS
};
515 /* Used by functions that have an unpredicated form and a _z predicated
517 static const predication_index preds_z_or_none
[] = {
518 PRED_z
, PRED_none
, NUM_PREDS
521 /* Used by (mostly predicate) functions that only support "_z" predication. */
522 static const predication_index preds_z
[] = { PRED_z
, NUM_PREDS
};
524 /* A list of all SVE ACLE functions. */
525 static CONSTEXPR
const function_group_info function_groups
[] = {
526 #define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \
527 { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, preds_##PREDS, \
528 REQUIRED_EXTENSIONS | AARCH64_FL_SVE },
529 #include "aarch64-sve-builtins.def"
532 /* The scalar type associated with each vector type. */
533 extern GTY(()) tree scalar_types
[NUM_VECTOR_TYPES
];
534 tree scalar_types
[NUM_VECTOR_TYPES
];
536 /* The single-predicate and single-vector types, with their built-in
537 "__SV..._t" name. Allow an index of NUM_VECTOR_TYPES, which always
538 yields a null tree. */
539 static GTY(()) tree abi_vector_types
[NUM_VECTOR_TYPES
+ 1];
541 /* Same, but with the arm_sve.h "sv..._t" name. */
542 extern GTY(()) tree acle_vector_types
[MAX_TUPLE_SIZE
][NUM_VECTOR_TYPES
+ 1];
543 tree acle_vector_types
[MAX_TUPLE_SIZE
][NUM_VECTOR_TYPES
+ 1];
545 /* The svpattern enum type. */
546 extern GTY(()) tree acle_svpattern
;
549 /* The svprfop enum type. */
550 extern GTY(()) tree acle_svprfop
;
553 /* The list of all registered function decls, indexed by code. */
554 static GTY(()) vec
<registered_function
*, va_gc
> *registered_functions
;
556 /* All registered function decls, hashed on the function_instance
557 that they implement. This is used for looking up implementations of
558 overloaded functions. */
559 static hash_table
<registered_function_hasher
> *function_table
;
561 /* True if we've already complained about attempts to use functions
562 when the required extension is disabled. */
563 static bool reported_missing_extension_p
;
565 /* True if we've already complained about attempts to use functions
566 which require registers that are missing. */
567 static bool reported_missing_registers_p
;
569 /* Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors
570 and NUM_PR SVE predicates. MANGLED_NAME, if nonnull, is the ABI-defined
571 mangling of the type. ACLE_NAME is the <arm_sve.h> name of the type. */
573 add_sve_type_attribute (tree type
, unsigned int num_zr
, unsigned int num_pr
,
574 const char *mangled_name
, const char *acle_name
)
576 tree mangled_name_tree
577 = (mangled_name
? get_identifier (mangled_name
) : NULL_TREE
);
579 tree value
= tree_cons (NULL_TREE
, get_identifier (acle_name
), NULL_TREE
);
580 value
= tree_cons (NULL_TREE
, mangled_name_tree
, value
);
581 value
= tree_cons (NULL_TREE
, size_int (num_pr
), value
);
582 value
= tree_cons (NULL_TREE
, size_int (num_zr
), value
);
583 TYPE_ATTRIBUTES (type
) = tree_cons (get_identifier ("SVE type"), value
,
584 TYPE_ATTRIBUTES (type
));
587 /* If TYPE is an ABI-defined SVE type, return its attribute descriptor,
588 otherwise return null. */
590 lookup_sve_type_attribute (const_tree type
)
592 if (type
== error_mark_node
)
594 return lookup_attribute ("SVE type", TYPE_ATTRIBUTES (type
));
597 /* Force TYPE to be a sizeless type. */
599 make_type_sizeless (tree type
)
601 TYPE_ATTRIBUTES (type
) = tree_cons (get_identifier ("SVE sizeless type"),
602 NULL_TREE
, TYPE_ATTRIBUTES (type
));
605 /* Return true if TYPE is a sizeless type. */
607 sizeless_type_p (const_tree type
)
609 if (type
== error_mark_node
)
611 return lookup_attribute ("SVE sizeless type", TYPE_ATTRIBUTES (type
));
614 /* Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading
617 matches_type_p (const_tree model_type
, const_tree candidate
)
619 if (VECTOR_TYPE_P (model_type
))
621 if (!VECTOR_TYPE_P (candidate
)
622 || maybe_ne (TYPE_VECTOR_SUBPARTS (model_type
),
623 TYPE_VECTOR_SUBPARTS (candidate
))
624 || TYPE_MODE (model_type
) != TYPE_MODE (candidate
))
627 model_type
= TREE_TYPE (model_type
);
628 candidate
= TREE_TYPE (candidate
);
630 return (candidate
!= error_mark_node
631 && TYPE_MAIN_VARIANT (model_type
) == TYPE_MAIN_VARIANT (candidate
));
634 /* If TYPE is a valid SVE element type, return the corresponding type
635 suffix, otherwise return NUM_TYPE_SUFFIXES. */
636 static type_suffix_index
637 find_type_suffix_for_scalar_type (const_tree type
)
639 /* A linear search should be OK here, since the code isn't hot and
640 the number of types is only small. */
641 for (unsigned int suffix_i
= 0; suffix_i
< NUM_TYPE_SUFFIXES
; ++suffix_i
)
642 if (!type_suffixes
[suffix_i
].bool_p
)
644 vector_type_index vector_i
= type_suffixes
[suffix_i
].vector_type
;
645 if (matches_type_p (scalar_types
[vector_i
], type
))
646 return type_suffix_index (suffix_i
);
648 return NUM_TYPE_SUFFIXES
;
651 /* Report an error against LOCATION that the user has tried to use
652 function FNDECL when extension EXTENSION is disabled. */
654 report_missing_extension (location_t location
, tree fndecl
,
655 const char *extension
)
657 /* Avoid reporting a slew of messages for a single oversight. */
658 if (reported_missing_extension_p
)
661 error_at (location
, "ACLE function %qD requires ISA extension %qs",
663 inform (location
, "you can enable %qs using the command-line"
664 " option %<-march%>, or by using the %<target%>"
665 " attribute or pragma", extension
);
666 reported_missing_extension_p
= true;
669 /* Check whether the registers required by SVE function fndecl are available.
670 Report an error against LOCATION and return false if not. */
672 check_required_registers (location_t location
, tree fndecl
)
674 /* Avoid reporting a slew of messages for a single oversight. */
675 if (reported_missing_registers_p
)
678 if (TARGET_GENERAL_REGS_ONLY
)
680 /* SVE registers are not usable when -mgeneral-regs-only option
683 "ACLE function %qD is incompatible with the use of %qs",
684 fndecl
, "-mgeneral-regs-only");
685 reported_missing_registers_p
= true;
692 /* Check whether all the AARCH64_FL_* values in REQUIRED_EXTENSIONS are
693 enabled, given that those extensions are required for function FNDECL.
694 Report an error against LOCATION if not. */
696 check_required_extensions (location_t location
, tree fndecl
,
697 aarch64_feature_flags required_extensions
)
699 auto missing_extensions
= required_extensions
& ~aarch64_isa_flags
;
700 if (missing_extensions
== 0)
701 return check_required_registers (location
, fndecl
);
703 static const struct {
704 aarch64_feature_flags flag
;
707 #define AARCH64_OPT_EXTENSION(EXT_NAME, IDENT, C, D, E, F) \
708 { AARCH64_FL_##IDENT, EXT_NAME },
709 #include "aarch64-option-extensions.def"
712 for (unsigned int i
= 0; i
< ARRAY_SIZE (extensions
); ++i
)
713 if (missing_extensions
& extensions
[i
].flag
)
715 report_missing_extension (location
, fndecl
, extensions
[i
].name
);
721 /* Report that LOCATION has a call to FNDECL in which argument ARGNO
722 was not an integer constant expression. ARGNO counts from zero. */
724 report_non_ice (location_t location
, tree fndecl
, unsigned int argno
)
726 error_at (location
, "argument %d of %qE must be an integer constant"
727 " expression", argno
+ 1, fndecl
);
730 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
731 the value ACTUAL, whereas the function requires a value in the range
732 [MIN, MAX]. ARGNO counts from zero. */
734 report_out_of_range (location_t location
, tree fndecl
, unsigned int argno
,
735 HOST_WIDE_INT actual
, HOST_WIDE_INT min
,
738 error_at (location
, "passing %wd to argument %d of %qE, which expects"
739 " a value in the range [%wd, %wd]", actual
, argno
+ 1, fndecl
,
743 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
744 the value ACTUAL, whereas the function requires either VALUE0 or
745 VALUE1. ARGNO counts from zero. */
747 report_neither_nor (location_t location
, tree fndecl
, unsigned int argno
,
748 HOST_WIDE_INT actual
, HOST_WIDE_INT value0
,
749 HOST_WIDE_INT value1
)
751 error_at (location
, "passing %wd to argument %d of %qE, which expects"
752 " either %wd or %wd", actual
, argno
+ 1, fndecl
, value0
, value1
);
755 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
756 the value ACTUAL, whereas the function requires one of VALUE0..3.
757 ARGNO counts from zero. */
759 report_not_one_of (location_t location
, tree fndecl
, unsigned int argno
,
760 HOST_WIDE_INT actual
, HOST_WIDE_INT value0
,
761 HOST_WIDE_INT value1
, HOST_WIDE_INT value2
,
762 HOST_WIDE_INT value3
)
764 error_at (location
, "passing %wd to argument %d of %qE, which expects"
765 " %wd, %wd, %wd or %wd", actual
, argno
+ 1, fndecl
, value0
, value1
,
769 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
770 the value ACTUAL, whereas the function requires a valid value of
771 enum type ENUMTYPE. ARGNO counts from zero. */
773 report_not_enum (location_t location
, tree fndecl
, unsigned int argno
,
774 HOST_WIDE_INT actual
, tree enumtype
)
776 error_at (location
, "passing %wd to argument %d of %qE, which expects"
777 " a valid %qT value", actual
, argno
+ 1, fndecl
, enumtype
);
780 /* Return a hash code for a function_instance. */
782 function_instance::hash () const
785 /* BASE uniquely determines BASE_NAME, so we don't need to hash both. */
788 h
.add_int (mode_suffix_id
);
789 h
.add_int (type_suffix_ids
[0]);
790 h
.add_int (type_suffix_ids
[1]);
795 /* Return a set of CP_* flags that describe what the function could do,
796 taking the command-line flags into account. */
798 function_instance::call_properties () const
800 unsigned int flags
= base
->call_properties (*this);
802 /* -fno-trapping-math means that we can assume any FP exceptions
803 are not user-visible. */
804 if (!flag_trapping_math
)
805 flags
&= ~CP_RAISE_FP_EXCEPTIONS
;
810 /* Return true if calls to the function could read some form of
813 function_instance::reads_global_state_p () const
815 unsigned int flags
= call_properties ();
817 /* Preserve any dependence on rounding mode, flush to zero mode, etc.
818 There is currently no way of turning this off; in particular,
819 -fno-rounding-math (which is the default) means that we should make
820 the usual assumptions about rounding mode, which for intrinsics means
821 acting as the instructions do. */
822 if (flags
& CP_READ_FPCR
)
825 /* Handle direct reads of global state. */
826 return flags
& (CP_READ_MEMORY
| CP_READ_FFR
);
829 /* Return true if calls to the function could modify some form of
832 function_instance::modifies_global_state_p () const
834 unsigned int flags
= call_properties ();
836 /* Preserve any exception state written back to the FPCR,
837 unless -fno-trapping-math says this is unnecessary. */
838 if (flags
& CP_RAISE_FP_EXCEPTIONS
)
841 /* Treat prefetches as modifying global state, since that's the
842 only means we have of keeping them in their correct position. */
843 if (flags
& CP_PREFETCH_MEMORY
)
846 /* Handle direct modifications of global state. */
847 return flags
& (CP_WRITE_MEMORY
| CP_WRITE_FFR
);
850 /* Return true if calls to the function could raise a signal. */
852 function_instance::could_trap_p () const
854 unsigned int flags
= call_properties ();
856 /* Handle functions that could raise SIGFPE. */
857 if (flags
& CP_RAISE_FP_EXCEPTIONS
)
860 /* Handle functions that could raise SIGBUS or SIGSEGV. */
861 if (flags
& (CP_READ_MEMORY
| CP_WRITE_MEMORY
))
868 registered_function_hasher::hash (value_type value
)
870 return value
->instance
.hash ();
874 registered_function_hasher::equal (value_type value
, const compare_type
&key
)
876 return value
->instance
== key
;
879 sve_switcher::sve_switcher ()
880 : aarch64_simd_switcher (AARCH64_FL_F16
| AARCH64_FL_SVE
)
882 /* Changing the ISA flags and have_regs_of_mode should be enough here.
883 We shouldn't need to pay the compile-time cost of a full target
885 m_old_maximum_field_alignment
= maximum_field_alignment
;
886 maximum_field_alignment
= 0;
888 memcpy (m_old_have_regs_of_mode
, have_regs_of_mode
,
889 sizeof (have_regs_of_mode
));
890 for (int i
= 0; i
< NUM_MACHINE_MODES
; ++i
)
891 if (aarch64_sve_mode_p ((machine_mode
) i
))
892 have_regs_of_mode
[i
] = true;
895 sve_switcher::~sve_switcher ()
897 memcpy (have_regs_of_mode
, m_old_have_regs_of_mode
,
898 sizeof (have_regs_of_mode
));
899 maximum_field_alignment
= m_old_maximum_field_alignment
;
902 function_builder::function_builder ()
904 m_overload_type
= build_function_type (void_type_node
, void_list_node
);
905 m_direct_overloads
= lang_GNU_CXX ();
906 gcc_obstack_init (&m_string_obstack
);
909 function_builder::~function_builder ()
911 obstack_free (&m_string_obstack
, NULL
);
914 /* Add NAME to the end of the function name being built. */
916 function_builder::append_name (const char *name
)
918 obstack_grow (&m_string_obstack
, name
, strlen (name
));
921 /* Zero-terminate and complete the function name being built. */
923 function_builder::finish_name ()
925 obstack_1grow (&m_string_obstack
, 0);
926 return (char *) obstack_finish (&m_string_obstack
);
929 /* Return the overloaded or full function name for INSTANCE; OVERLOADED_P
930 selects which. Allocate the string on m_string_obstack; the caller
931 must use obstack_free to free it after use. */
933 function_builder::get_name (const function_instance
&instance
,
936 append_name (instance
.base_name
);
938 switch (instance
.displacement_units ())
944 append_name ("_offset");
948 append_name ("_index");
952 append_name ("_vnum");
956 append_name (instance
.mode_suffix ().string
);
957 for (unsigned int i
= 0; i
< 2; ++i
)
958 if (!overloaded_p
|| instance
.shape
->explicit_type_suffix_p (i
))
959 append_name (instance
.type_suffix (i
).string
);
960 append_name (pred_suffixes
[instance
.pred
]);
961 return finish_name ();
964 /* Add attribute NAME to ATTRS. */
966 add_attribute (const char *name
, tree attrs
)
968 return tree_cons (get_identifier (name
), NULL_TREE
, attrs
);
971 /* Return the appropriate function attributes for INSTANCE. */
973 function_builder::get_attributes (const function_instance
&instance
)
975 tree attrs
= NULL_TREE
;
977 if (!instance
.modifies_global_state_p ())
979 if (instance
.reads_global_state_p ())
980 attrs
= add_attribute ("pure", attrs
);
982 attrs
= add_attribute ("const", attrs
);
985 if (!flag_non_call_exceptions
|| !instance
.could_trap_p ())
986 attrs
= add_attribute ("nothrow", attrs
);
988 return add_attribute ("leaf", attrs
);
991 /* Add a function called NAME with type FNTYPE and attributes ATTRS.
992 INSTANCE describes what the function does and OVERLOADED_P indicates
993 whether it is overloaded. REQUIRED_EXTENSIONS are the set of
994 architecture extensions that the function requires. */
995 registered_function
&
996 function_builder::add_function (const function_instance
&instance
,
997 const char *name
, tree fntype
, tree attrs
,
998 aarch64_feature_flags required_extensions
,
1002 unsigned int code
= vec_safe_length (registered_functions
);
1003 code
= (code
<< AARCH64_BUILTIN_SHIFT
) | AARCH64_BUILTIN_SVE
;
1005 /* We need to be able to generate placeholders to enusre that we have a
1006 consistent numbering scheme for function codes between the C and C++
1007 frontends, so that everything ties up in LTO.
1009 Currently, tree-streamer-in.cc:unpack_ts_function_decl_value_fields
1010 validates that tree nodes returned by TARGET_BUILTIN_DECL are non-NULL and
1011 some node other than error_mark_node. This is a holdover from when builtin
1012 decls were streamed by code rather than by value.
1014 Ultimately, we should be able to remove this validation of BUILT_IN_MD
1015 nodes and remove the target hook. For now, however, we need to appease the
1016 validation and return a non-NULL, non-error_mark_node node, so we
1017 arbitrarily choose integer_zero_node. */
1018 tree decl
= placeholder_p
1020 : simulate_builtin_function_decl (input_location
, name
, fntype
,
1023 registered_function
&rfn
= *ggc_alloc
<registered_function
> ();
1024 rfn
.instance
= instance
;
1026 rfn
.required_extensions
= required_extensions
;
1027 rfn
.overloaded_p
= overloaded_p
;
1028 vec_safe_push (registered_functions
, &rfn
);
1033 /* Add a built-in function for INSTANCE, with the argument types given
1034 by ARGUMENT_TYPES and the return type given by RETURN_TYPE.
1035 REQUIRED_EXTENSIONS are the set of architecture extensions that the
1036 function requires. FORCE_DIRECT_OVERLOADS is true if there is a
1037 one-to-one mapping between "short" and "full" names, and if standard
1038 overload resolution therefore isn't necessary. */
1041 add_unique_function (const function_instance
&instance
,
1043 vec
<tree
> &argument_types
,
1044 aarch64_feature_flags required_extensions
,
1045 bool force_direct_overloads
)
1047 /* Add the function under its full (unique) name. */
1048 char *name
= get_name (instance
, false);
1049 tree fntype
= build_function_type_array (return_type
,
1050 argument_types
.length (),
1051 argument_types
.address ());
1052 tree attrs
= get_attributes (instance
);
1053 registered_function
&rfn
= add_function (instance
, name
, fntype
, attrs
,
1054 required_extensions
, false, false);
1056 /* Enter the function into the hash table. */
1057 hashval_t hash
= instance
.hash ();
1058 registered_function
**rfn_slot
1059 = function_table
->find_slot_with_hash (instance
, hash
, INSERT
);
1060 gcc_assert (!*rfn_slot
);
1063 /* Also add the function under its overloaded alias, if we want
1064 a separate decl for each instance of an overloaded function. */
1065 char *overload_name
= get_name (instance
, true);
1066 if (strcmp (name
, overload_name
) != 0)
1068 /* Attribute lists shouldn't be shared. */
1069 tree attrs
= get_attributes (instance
);
1070 bool placeholder_p
= !(m_direct_overloads
|| force_direct_overloads
);
1071 add_function (instance
, overload_name
, fntype
, attrs
,
1072 required_extensions
, false, placeholder_p
);
1075 obstack_free (&m_string_obstack
, name
);
1078 /* Add one function decl for INSTANCE, to be used with manual overload
1079 resolution. REQUIRED_EXTENSIONS are the set of architecture extensions
1080 that the function requires.
1082 For simplicity, deal with duplicate attempts to add the same function,
1083 including cases in which the new function requires more features than
1084 the original one did. In that case we'll check whether the required
1085 features are available as part of resolving the function to the
1086 relevant unique function. */
1089 add_overloaded_function (const function_instance
&instance
,
1090 aarch64_feature_flags required_extensions
)
1092 char *name
= get_name (instance
, true);
1093 if (registered_function
**map_value
= m_overload_names
.get (name
))
1095 gcc_assert ((*map_value
)->instance
== instance
1096 && ((*map_value
)->required_extensions
1097 & ~required_extensions
) == 0);
1098 obstack_free (&m_string_obstack
, name
);
1102 registered_function
&rfn
1103 = add_function (instance
, name
, m_overload_type
, NULL_TREE
,
1104 required_extensions
, true, m_direct_overloads
);
1105 m_overload_names
.put (name
, &rfn
);
1109 /* If we are using manual overload resolution, add one function decl
1110 for each overloaded function in GROUP. Take the function base name
1111 from GROUP and the mode from MODE. */
1113 function_builder::add_overloaded_functions (const function_group_info
&group
,
1114 mode_suffix_index mode
)
1116 unsigned int explicit_type0
= (*group
.shape
)->explicit_type_suffix_p (0);
1117 unsigned int explicit_type1
= (*group
.shape
)->explicit_type_suffix_p (1);
1118 for (unsigned int pi
= 0; group
.preds
[pi
] != NUM_PREDS
; ++pi
)
1120 if (!explicit_type0
&& !explicit_type1
)
1122 /* Deal with the common case in which there is one overloaded
1123 function for all type combinations. */
1124 function_instance
instance (group
.base_name
, *group
.base
,
1125 *group
.shape
, mode
, types_none
[0],
1127 add_overloaded_function (instance
, group
.required_extensions
);
1130 for (unsigned int ti
= 0; group
.types
[ti
][0] != NUM_TYPE_SUFFIXES
;
1133 /* Stub out the types that are determined by overload
1135 type_suffix_pair types
= {
1136 explicit_type0
? group
.types
[ti
][0] : NUM_TYPE_SUFFIXES
,
1137 explicit_type1
? group
.types
[ti
][1] : NUM_TYPE_SUFFIXES
1139 function_instance
instance (group
.base_name
, *group
.base
,
1140 *group
.shape
, mode
, types
,
1142 add_overloaded_function (instance
, group
.required_extensions
);
1147 /* Register all the functions in GROUP. */
1149 function_builder::register_function_group (const function_group_info
&group
)
1151 (*group
.shape
)->build (*this, group
);
1154 function_call_info::function_call_info (location_t location_in
,
1155 const function_instance
&instance_in
,
1157 : function_instance (instance_in
), location (location_in
), fndecl (fndecl_in
)
1161 function_resolver::function_resolver (location_t location
,
1162 const function_instance
&instance
,
1163 tree fndecl
, vec
<tree
, va_gc
> &arglist
)
1164 : function_call_info (location
, instance
, fndecl
), m_arglist (arglist
)
1168 /* Return the vector type associated with type suffix TYPE. */
1170 function_resolver::get_vector_type (type_suffix_index type
)
1172 return acle_vector_types
[0][type_suffixes
[type
].vector_type
];
1175 /* Return the <stdint.h> name associated with TYPE. Using the <stdint.h>
1176 name should be more user-friendly than the underlying canonical type,
1177 since it makes the signedness and bitwidth explicit. */
1179 function_resolver::get_scalar_type_name (type_suffix_index type
)
1181 return vector_types
[type_suffixes
[type
].vector_type
].acle_name
+ 2;
1184 /* Return the type of argument I, or error_mark_node if it isn't
1187 function_resolver::get_argument_type (unsigned int i
)
1189 tree arg
= m_arglist
[i
];
1190 return arg
== error_mark_node
? arg
: TREE_TYPE (arg
);
1193 /* Return true if argument I is some form of scalar value. */
1195 function_resolver::scalar_argument_p (unsigned int i
)
1197 tree type
= get_argument_type (i
);
1198 return (INTEGRAL_TYPE_P (type
)
1199 /* Allow pointer types, leaving the frontend to warn where
1201 || POINTER_TYPE_P (type
)
1202 || SCALAR_FLOAT_TYPE_P (type
));
1205 /* Report that the function has no form that takes type suffix TYPE.
1206 Return error_mark_node. */
1208 function_resolver::report_no_such_form (type_suffix_index type
)
1210 error_at (location
, "%qE has no form that takes %qT arguments",
1211 fndecl
, get_vector_type (type
));
1212 return error_mark_node
;
1215 /* Silently check whether there is an instance of the function with the
1216 mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1.
1217 Return its function decl if so, otherwise return null. */
1219 function_resolver::lookup_form (mode_suffix_index mode
,
1220 type_suffix_index type0
,
1221 type_suffix_index type1
)
1223 type_suffix_pair types
= { type0
, type1
};
1224 function_instance
instance (base_name
, base
, shape
, mode
, types
, pred
);
1225 registered_function
*rfn
1226 = function_table
->find_with_hash (instance
, instance
.hash ());
1227 return rfn
? rfn
->decl
: NULL_TREE
;
1230 /* Resolve the function to one with the mode suffix given by MODE and the
1231 type suffixes given by TYPE0 and TYPE1. Return its function decl on
1232 success, otherwise report an error and return error_mark_node. */
1234 function_resolver::resolve_to (mode_suffix_index mode
,
1235 type_suffix_index type0
,
1236 type_suffix_index type1
)
1238 tree res
= lookup_form (mode
, type0
, type1
);
1241 if (type1
== NUM_TYPE_SUFFIXES
)
1242 return report_no_such_form (type0
);
1243 if (type0
== type_suffix_ids
[0])
1244 return report_no_such_form (type1
);
1245 /* To be filled in when we have other cases. */
1251 /* Require argument ARGNO to be a 32-bit or 64-bit scalar integer type.
1252 Return the associated type suffix on success, otherwise report an
1253 error and return NUM_TYPE_SUFFIXES. */
1255 function_resolver::infer_integer_scalar_type (unsigned int argno
)
1257 tree actual
= get_argument_type (argno
);
1258 if (actual
== error_mark_node
)
1259 return NUM_TYPE_SUFFIXES
;
1261 /* Allow enums and booleans to decay to integers, for compatibility
1262 with C++ overloading rules. */
1263 if (INTEGRAL_TYPE_P (actual
))
1265 bool uns_p
= TYPE_UNSIGNED (actual
);
1266 /* Honor the usual integer promotions, so that resolution works
1267 in the same way as for C++. */
1268 if (TYPE_PRECISION (actual
) < 32)
1269 return TYPE_SUFFIX_s32
;
1270 if (TYPE_PRECISION (actual
) == 32)
1271 return uns_p
? TYPE_SUFFIX_u32
: TYPE_SUFFIX_s32
;
1272 if (TYPE_PRECISION (actual
) == 64)
1273 return uns_p
? TYPE_SUFFIX_u64
: TYPE_SUFFIX_s64
;
1276 error_at (location
, "passing %qT to argument %d of %qE, which expects"
1277 " a 32-bit or 64-bit integer type", actual
, argno
+ 1, fndecl
);
1278 return NUM_TYPE_SUFFIXES
;
1281 /* Require argument ARGNO to be a pointer to a scalar type that has a
1282 corresponding type suffix. Return that type suffix on success,
1283 otherwise report an error and return NUM_TYPE_SUFFIXES.
1284 GATHER_SCATTER_P is true if the function is a gather/scatter
1285 operation, and so requires a pointer to 32-bit or 64-bit data. */
1287 function_resolver::infer_pointer_type (unsigned int argno
,
1288 bool gather_scatter_p
)
1290 tree actual
= get_argument_type (argno
);
1291 if (actual
== error_mark_node
)
1292 return NUM_TYPE_SUFFIXES
;
1294 if (TREE_CODE (actual
) != POINTER_TYPE
)
1296 error_at (location
, "passing %qT to argument %d of %qE, which"
1297 " expects a pointer type", actual
, argno
+ 1, fndecl
);
1298 if (VECTOR_TYPE_P (actual
) && gather_scatter_p
)
1299 inform (location
, "an explicit type suffix is needed"
1300 " when using a vector of base addresses");
1301 return NUM_TYPE_SUFFIXES
;
1304 tree target
= TREE_TYPE (actual
);
1305 type_suffix_index type
= find_type_suffix_for_scalar_type (target
);
1306 if (type
== NUM_TYPE_SUFFIXES
)
1308 error_at (location
, "passing %qT to argument %d of %qE, but %qT is not"
1309 " a valid SVE element type", actual
, argno
+ 1, fndecl
,
1310 build_qualified_type (target
, 0));
1311 return NUM_TYPE_SUFFIXES
;
1313 unsigned int bits
= type_suffixes
[type
].element_bits
;
1314 if (gather_scatter_p
&& bits
!= 32 && bits
!= 64)
1316 error_at (location
, "passing %qT to argument %d of %qE, which"
1317 " expects a pointer to 32-bit or 64-bit elements",
1318 actual
, argno
+ 1, fndecl
);
1319 return NUM_TYPE_SUFFIXES
;
1325 /* Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS
1326 vectors; NUM_VECTORS is 1 for the former. Return the associated type
1327 suffix on success, using TYPE_SUFFIX_b for predicates. Report an error
1328 and return NUM_TYPE_SUFFIXES on failure. */
1330 function_resolver::infer_vector_or_tuple_type (unsigned int argno
,
1331 unsigned int num_vectors
)
1333 tree actual
= get_argument_type (argno
);
1334 if (actual
== error_mark_node
)
1335 return NUM_TYPE_SUFFIXES
;
1337 /* A linear search should be OK here, since the code isn't hot and
1338 the number of types is only small. */
1339 for (unsigned int size_i
= 0; size_i
< MAX_TUPLE_SIZE
; ++size_i
)
1340 for (unsigned int suffix_i
= 0; suffix_i
< NUM_TYPE_SUFFIXES
; ++suffix_i
)
1342 vector_type_index type_i
= type_suffixes
[suffix_i
].vector_type
;
1343 tree type
= acle_vector_types
[size_i
][type_i
];
1344 if (type
&& matches_type_p (type
, actual
))
1346 if (size_i
+ 1 == num_vectors
)
1347 return type_suffix_index (suffix_i
);
1349 if (num_vectors
== 1)
1350 error_at (location
, "passing %qT to argument %d of %qE, which"
1351 " expects a single SVE vector rather than a tuple",
1352 actual
, argno
+ 1, fndecl
);
1353 else if (size_i
== 0 && type_i
!= VECTOR_TYPE_svbool_t
)
1354 /* num_vectors is always != 1, so the singular isn't needed. */
1355 error_n (location
, num_vectors
, "%qT%d%qE%d",
1356 "passing single vector %qT to argument %d"
1357 " of %qE, which expects a tuple of %d vectors",
1358 actual
, argno
+ 1, fndecl
, num_vectors
);
1360 /* num_vectors is always != 1, so the singular isn't needed. */
1361 error_n (location
, num_vectors
, "%qT%d%qE%d",
1362 "passing %qT to argument %d of %qE, which"
1363 " expects a tuple of %d vectors", actual
, argno
+ 1,
1364 fndecl
, num_vectors
);
1365 return NUM_TYPE_SUFFIXES
;
1369 if (num_vectors
== 1)
1370 error_at (location
, "passing %qT to argument %d of %qE, which"
1371 " expects an SVE vector type", actual
, argno
+ 1, fndecl
);
1373 error_at (location
, "passing %qT to argument %d of %qE, which"
1374 " expects an SVE tuple type", actual
, argno
+ 1, fndecl
);
1375 return NUM_TYPE_SUFFIXES
;
1378 /* Require argument ARGNO to have some form of vector type. Return the
1379 associated type suffix on success, using TYPE_SUFFIX_b for predicates.
1380 Report an error and return NUM_TYPE_SUFFIXES on failure. */
1382 function_resolver::infer_vector_type (unsigned int argno
)
1384 return infer_vector_or_tuple_type (argno
, 1);
1387 /* Like infer_vector_type, but also require the type to be integral. */
1389 function_resolver::infer_integer_vector_type (unsigned int argno
)
1391 type_suffix_index type
= infer_vector_type (argno
);
1392 if (type
== NUM_TYPE_SUFFIXES
)
1395 if (!type_suffixes
[type
].integer_p
)
1397 error_at (location
, "passing %qT to argument %d of %qE, which"
1398 " expects a vector of integers", get_argument_type (argno
),
1400 return NUM_TYPE_SUFFIXES
;
1406 /* Like infer_vector_type, but also require the type to be an unsigned
1409 function_resolver::infer_unsigned_vector_type (unsigned int argno
)
1411 type_suffix_index type
= infer_vector_type (argno
);
1412 if (type
== NUM_TYPE_SUFFIXES
)
1415 if (!type_suffixes
[type
].unsigned_p
)
1417 error_at (location
, "passing %qT to argument %d of %qE, which"
1418 " expects a vector of unsigned integers",
1419 get_argument_type (argno
), argno
+ 1, fndecl
);
1420 return NUM_TYPE_SUFFIXES
;
1426 /* Like infer_vector_type, but also require the element size to be
1429 function_resolver::infer_sd_vector_type (unsigned int argno
)
1431 type_suffix_index type
= infer_vector_type (argno
);
1432 if (type
== NUM_TYPE_SUFFIXES
)
1435 unsigned int bits
= type_suffixes
[type
].element_bits
;
1436 if (bits
!= 32 && bits
!= 64)
1438 error_at (location
, "passing %qT to argument %d of %qE, which"
1439 " expects a vector of 32-bit or 64-bit elements",
1440 get_argument_type (argno
), argno
+ 1, fndecl
);
1441 return NUM_TYPE_SUFFIXES
;
1447 /* If the function operates on tuples of vectors, require argument ARGNO to be
1448 a tuple with the appropriate number of vectors, otherwise require it to be
1449 a single vector. Return the associated type suffix on success, using
1450 TYPE_SUFFIX_b for predicates. Report an error and return NUM_TYPE_SUFFIXES
1453 function_resolver::infer_tuple_type (unsigned int argno
)
1455 return infer_vector_or_tuple_type (argno
, vectors_per_tuple ());
1458 /* Require argument ARGNO to be a vector or scalar argument. Return true
1459 if it is, otherwise report an appropriate error. */
1461 function_resolver::require_vector_or_scalar_type (unsigned int argno
)
1463 tree actual
= get_argument_type (argno
);
1464 if (actual
== error_mark_node
)
1467 if (!scalar_argument_p (argno
) && !VECTOR_TYPE_P (actual
))
1469 error_at (location
, "passing %qT to argument %d of %qE, which"
1470 " expects a vector or scalar type", actual
, argno
+ 1, fndecl
);
1477 /* Require argument ARGNO to have vector type TYPE, in cases where this
1478 requirement holds for all uses of the function. Return true if the
1479 argument has the right form, otherwise report an appropriate error. */
1481 function_resolver::require_vector_type (unsigned int argno
,
1482 vector_type_index type
)
1484 tree expected
= acle_vector_types
[0][type
];
1485 tree actual
= get_argument_type (argno
);
1486 if (actual
== error_mark_node
)
1489 if (!matches_type_p (expected
, actual
))
1491 error_at (location
, "passing %qT to argument %d of %qE, which"
1492 " expects %qT", actual
, argno
+ 1, fndecl
, expected
);
1498 /* Like require_vector_type, but TYPE is inferred from previous arguments
1499 rather than being a fixed part of the function signature. This changes
1500 the nature of the error messages. */
1502 function_resolver::require_matching_vector_type (unsigned int argno
,
1503 type_suffix_index type
)
1505 type_suffix_index new_type
= infer_vector_type (argno
);
1506 if (new_type
== NUM_TYPE_SUFFIXES
)
1509 if (type
!= new_type
)
1511 error_at (location
, "passing %qT to argument %d of %qE, but"
1512 " previous arguments had type %qT",
1513 get_vector_type (new_type
), argno
+ 1, fndecl
,
1514 get_vector_type (type
));
1520 /* Require argument ARGNO to be a vector type with the following properties:
1522 - the type class must be the same as FIRST_TYPE's if EXPECTED_TCLASS
1523 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself.
1525 - the element size must be:
1527 - the same as FIRST_TYPE's if EXPECTED_BITS == SAME_SIZE
1528 - half of FIRST_TYPE's if EXPECTED_BITS == HALF_SIZE
1529 - a quarter of FIRST_TYPE's if EXPECTED_BITS == QUARTER_SIZE
1530 - EXPECTED_BITS itself otherwise
1532 Return true if the argument has the required type, otherwise report
1533 an appropriate error.
1535 FIRST_ARGNO is the first argument that is known to have type FIRST_TYPE.
1536 Usually it comes before ARGNO, but sometimes it is more natural to resolve
1537 arguments out of order.
1539 If the required properties depend on FIRST_TYPE then both FIRST_ARGNO and
1540 ARGNO contribute to the resolution process. If the required properties
1541 are fixed, only FIRST_ARGNO contributes to the resolution process.
1543 This function is a bit of a Swiss army knife. The complication comes
1544 from trying to give good error messages when FIRST_ARGNO and ARGNO are
1545 inconsistent, since either of them might be wrong. */
1546 bool function_resolver::
1547 require_derived_vector_type (unsigned int argno
,
1548 unsigned int first_argno
,
1549 type_suffix_index first_type
,
1550 type_class_index expected_tclass
,
1551 unsigned int expected_bits
)
1553 /* If the type needs to match FIRST_ARGNO exactly, use the preferred
1554 error message for that case. The VECTOR_TYPE_P test excludes tuple
1555 types, which we handle below instead. */
1556 bool both_vectors_p
= VECTOR_TYPE_P (get_argument_type (first_argno
));
1558 && expected_tclass
== SAME_TYPE_CLASS
1559 && expected_bits
== SAME_SIZE
)
1561 /* There's no need to resolve this case out of order. */
1562 gcc_assert (argno
> first_argno
);
1563 return require_matching_vector_type (argno
, first_type
);
1566 /* Use FIRST_TYPE to get the expected type class and element size. */
1567 type_class_index orig_expected_tclass
= expected_tclass
;
1568 if (expected_tclass
== NUM_TYPE_CLASSES
)
1569 expected_tclass
= type_suffixes
[first_type
].tclass
;
1571 unsigned int orig_expected_bits
= expected_bits
;
1572 if (expected_bits
== SAME_SIZE
)
1573 expected_bits
= type_suffixes
[first_type
].element_bits
;
1574 else if (expected_bits
== HALF_SIZE
)
1575 expected_bits
= type_suffixes
[first_type
].element_bits
/ 2;
1576 else if (expected_bits
== QUARTER_SIZE
)
1577 expected_bits
= type_suffixes
[first_type
].element_bits
/ 4;
1579 /* If the expected type doesn't depend on FIRST_TYPE at all,
1580 just check for the fixed choice of vector type. */
1581 if (expected_tclass
== orig_expected_tclass
1582 && expected_bits
== orig_expected_bits
)
1584 const type_suffix_info
&expected_suffix
1585 = type_suffixes
[find_type_suffix (expected_tclass
, expected_bits
)];
1586 return require_vector_type (argno
, expected_suffix
.vector_type
);
1589 /* Require the argument to be some form of SVE vector type,
1590 without being specific about the type of vector we want. */
1591 type_suffix_index actual_type
= infer_vector_type (argno
);
1592 if (actual_type
== NUM_TYPE_SUFFIXES
)
1595 /* Exit now if we got the right type. */
1596 bool tclass_ok_p
= (type_suffixes
[actual_type
].tclass
== expected_tclass
);
1597 bool size_ok_p
= (type_suffixes
[actual_type
].element_bits
== expected_bits
);
1598 if (tclass_ok_p
&& size_ok_p
)
1601 /* First look for cases in which the actual type contravenes a fixed
1602 size requirement, without having to refer to FIRST_TYPE. */
1603 if (!size_ok_p
&& expected_bits
== orig_expected_bits
)
1605 error_at (location
, "passing %qT to argument %d of %qE, which"
1606 " expects a vector of %d-bit elements",
1607 get_vector_type (actual_type
), argno
+ 1, fndecl
,
1612 /* Likewise for a fixed type class requirement. This is only ever
1613 needed for signed and unsigned types, so don't create unnecessary
1614 translation work for other type classes. */
1615 if (!tclass_ok_p
&& orig_expected_tclass
== TYPE_signed
)
1617 error_at (location
, "passing %qT to argument %d of %qE, which"
1618 " expects a vector of signed integers",
1619 get_vector_type (actual_type
), argno
+ 1, fndecl
);
1622 if (!tclass_ok_p
&& orig_expected_tclass
== TYPE_unsigned
)
1624 error_at (location
, "passing %qT to argument %d of %qE, which"
1625 " expects a vector of unsigned integers",
1626 get_vector_type (actual_type
), argno
+ 1, fndecl
);
1630 /* Make sure that FIRST_TYPE itself is sensible before using it
1631 as a basis for an error message. */
1632 if (resolve_to (mode_suffix_id
, first_type
) == error_mark_node
)
1635 /* If the arguments have consistent type classes, but a link between
1636 the sizes has been broken, try to describe the error in those terms. */
1637 if (both_vectors_p
&& tclass_ok_p
&& orig_expected_bits
== SAME_SIZE
)
1639 if (argno
< first_argno
)
1641 std::swap (argno
, first_argno
);
1642 std::swap (actual_type
, first_type
);
1644 error_at (location
, "arguments %d and %d of %qE must have the"
1645 " same element size, but the values passed here have type"
1646 " %qT and %qT respectively", first_argno
+ 1, argno
+ 1,
1647 fndecl
, get_vector_type (first_type
),
1648 get_vector_type (actual_type
));
1652 /* Likewise in reverse: look for cases in which the sizes are consistent
1653 but a link between the type classes has been broken. */
1656 && orig_expected_tclass
== SAME_TYPE_CLASS
1657 && type_suffixes
[first_type
].integer_p
1658 && type_suffixes
[actual_type
].integer_p
)
1660 if (argno
< first_argno
)
1662 std::swap (argno
, first_argno
);
1663 std::swap (actual_type
, first_type
);
1665 error_at (location
, "arguments %d and %d of %qE must have the"
1666 " same signedness, but the values passed here have type"
1667 " %qT and %qT respectively", first_argno
+ 1, argno
+ 1,
1668 fndecl
, get_vector_type (first_type
),
1669 get_vector_type (actual_type
));
1673 /* The two arguments are wildly inconsistent. */
1674 type_suffix_index expected_type
1675 = find_type_suffix (expected_tclass
, expected_bits
);
1676 error_at (location
, "passing %qT instead of the expected %qT to argument"
1677 " %d of %qE, after passing %qT to argument %d",
1678 get_vector_type (actual_type
), get_vector_type (expected_type
),
1679 argno
+ 1, fndecl
, get_argument_type (first_argno
),
1684 /* Require argument ARGNO to match argument FIRST_ARGNO, which was inferred
1685 to be a pointer to a scalar element of type TYPE. */
1687 function_resolver::require_matching_pointer_type (unsigned int argno
,
1688 unsigned int first_argno
,
1689 type_suffix_index type
)
1691 type_suffix_index new_type
= infer_pointer_type (argno
);
1692 if (new_type
== NUM_TYPE_SUFFIXES
)
1695 if (type
!= new_type
)
1697 error_at (location
, "passing %qT to argument %d of %qE, but"
1698 " argument %d had type %qT", get_argument_type (argno
),
1699 argno
+ 1, fndecl
, first_argno
+ 1,
1700 get_argument_type (first_argno
));
1706 /* Require argument ARGNO to be a (possibly variable) scalar, using EXPECTED
1707 as the name of its expected type. Return true if the argument has the
1708 right form, otherwise report an appropriate error. */
1710 function_resolver::require_scalar_type (unsigned int argno
,
1711 const char *expected
)
1713 if (!scalar_argument_p (argno
))
1715 error_at (location
, "passing %qT to argument %d of %qE, which"
1716 " expects %qs", get_argument_type (argno
), argno
+ 1,
1723 /* Require argument ARGNO to be some form of pointer, without being specific
1724 about its target type. Return true if the argument has the right form,
1725 otherwise report an appropriate error. */
1727 function_resolver::require_pointer_type (unsigned int argno
)
1729 if (!scalar_argument_p (argno
))
1731 error_at (location
, "passing %qT to argument %d of %qE, which"
1732 " expects a scalar pointer", get_argument_type (argno
),
1739 /* Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE, and argument
1740 ARGNO should be consistent with it. Return true if it is, otherwise
1741 report an appropriate error. */
1742 bool function_resolver::
1743 require_matching_integer_scalar_type (unsigned int argno
,
1744 unsigned int first_argno
,
1745 type_suffix_index expected_type
)
1747 type_suffix_index actual_type
= infer_integer_scalar_type (argno
);
1748 if (actual_type
== NUM_TYPE_SUFFIXES
)
1751 if (actual_type
== expected_type
)
1754 error_at (location
, "call to %qE is ambiguous; argument %d has type"
1755 " %qs but argument %d has type %qs", fndecl
,
1756 first_argno
+ 1, get_scalar_type_name (expected_type
),
1757 argno
+ 1, get_scalar_type_name (actual_type
));
1761 /* Require argument ARGNO to be a (possibly variable) scalar, expecting it
1762 to have the following properties:
1764 - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS
1765 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself.
1767 - the element size must be the same as for type suffix 0 if EXPECTED_BITS
1768 is SAME_TYPE_SIZE, otherwise it must be EXPECTED_BITS itself.
1770 Return true if the argument is valid, otherwise report an appropriate error.
1772 Note that we don't check whether the scalar type actually has the required
1773 properties, since that's subject to implicit promotions and conversions.
1774 Instead we just use the expected properties to tune the error message. */
1775 bool function_resolver::
1776 require_derived_scalar_type (unsigned int argno
,
1777 type_class_index expected_tclass
,
1778 unsigned int expected_bits
)
1780 gcc_assert (expected_tclass
== SAME_TYPE_CLASS
1781 || expected_tclass
== TYPE_signed
1782 || expected_tclass
== TYPE_unsigned
);
1784 /* If the expected type doesn't depend on the type suffix at all,
1785 just check for the fixed choice of scalar type. */
1786 if (expected_tclass
!= SAME_TYPE_CLASS
&& expected_bits
!= SAME_SIZE
)
1788 type_suffix_index expected_type
1789 = find_type_suffix (expected_tclass
, expected_bits
);
1790 return require_scalar_type (argno
, get_scalar_type_name (expected_type
));
1793 if (scalar_argument_p (argno
))
1796 if (expected_tclass
== SAME_TYPE_CLASS
)
1797 /* It doesn't really matter whether the element is expected to be
1798 the same size as type suffix 0. */
1799 error_at (location
, "passing %qT to argument %d of %qE, which"
1800 " expects a scalar element", get_argument_type (argno
),
1803 /* It doesn't seem useful to distinguish between signed and unsigned
1805 error_at (location
, "passing %qT to argument %d of %qE, which"
1806 " expects a scalar integer", get_argument_type (argno
),
1811 /* Require argument ARGNO to be suitable for an integer constant expression.
1812 Return true if it is, otherwise report an appropriate error.
1814 function_checker checks whether the argument is actually constant and
1815 has a suitable range. The reason for distinguishing immediate arguments
1816 here is because it provides more consistent error messages than
1817 require_scalar_type would. */
1819 function_resolver::require_integer_immediate (unsigned int argno
)
1821 if (!scalar_argument_p (argno
))
1823 report_non_ice (location
, fndecl
, argno
);
1829 /* Require argument ARGNO to be a vector base in a gather-style address.
1830 Return its type on success, otherwise return NUM_VECTOR_TYPES. */
1832 function_resolver::infer_vector_base_type (unsigned int argno
)
1834 type_suffix_index type
= infer_vector_type (argno
);
1835 if (type
== NUM_TYPE_SUFFIXES
)
1836 return NUM_VECTOR_TYPES
;
1838 if (type
== TYPE_SUFFIX_u32
|| type
== TYPE_SUFFIX_u64
)
1839 return type_suffixes
[type
].vector_type
;
1841 error_at (location
, "passing %qT to argument %d of %qE, which"
1842 " expects %qs or %qs", get_argument_type (argno
),
1843 argno
+ 1, fndecl
, "svuint32_t", "svuint64_t");
1844 return NUM_VECTOR_TYPES
;
1847 /* Require argument ARGNO to be a vector displacement in a gather-style
1848 address. Return its type on success, otherwise return NUM_VECTOR_TYPES. */
1850 function_resolver::infer_vector_displacement_type (unsigned int argno
)
1852 type_suffix_index type
= infer_integer_vector_type (argno
);
1853 if (type
== NUM_TYPE_SUFFIXES
)
1854 return NUM_VECTOR_TYPES
;
1856 if (type_suffixes
[type
].integer_p
1857 && (type_suffixes
[type
].element_bits
== 32
1858 || type_suffixes
[type
].element_bits
== 64))
1859 return type_suffixes
[type
].vector_type
;
1861 error_at (location
, "passing %qT to argument %d of %qE, which"
1862 " expects a vector of 32-bit or 64-bit integers",
1863 get_argument_type (argno
), argno
+ 1, fndecl
);
1864 return NUM_VECTOR_TYPES
;
1867 /* Require argument ARGNO to be a vector displacement in a gather-style
1868 address. There are three possible uses:
1870 - for loading into elements of type TYPE (when LOAD_P is true)
1871 - for storing from elements of type TYPE (when LOAD_P is false)
1872 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES)
1874 The overloaded function's mode suffix determines the units of the
1875 displacement (bytes for "_offset", elements for "_index").
1877 Return the associated mode on success, otherwise report an error
1878 and return MODE_none. */
1880 function_resolver::resolve_sv_displacement (unsigned int argno
,
1881 type_suffix_index type
,
1884 if (type
== NUM_TYPE_SUFFIXES
)
1886 /* For prefetches, the base is a void pointer and the displacement
1887 can be any valid offset or index type. */
1888 vector_type_index displacement_vector_type
1889 = infer_vector_displacement_type (argno
);
1890 if (displacement_vector_type
== NUM_VECTOR_TYPES
)
1893 mode_suffix_index mode
= find_mode_suffix (NUM_VECTOR_TYPES
,
1894 displacement_vector_type
,
1895 displacement_units ());
1896 gcc_assert (mode
!= MODE_none
);
1900 unsigned int required_bits
= type_suffixes
[type
].element_bits
;
1901 if (required_bits
== 32
1902 && displacement_units () == UNITS_elements
1903 && !lookup_form (MODE_s32index
, type
)
1904 && !lookup_form (MODE_u32index
, type
))
1906 if (lookup_form (MODE_u32base_index
, type
))
1908 if (type_suffix_ids
[0] == NUM_TYPE_SUFFIXES
)
1910 gcc_assert (!load_p
);
1911 error_at (location
, "when storing %qT, %qE requires a vector"
1912 " base and a scalar index", get_vector_type (type
),
1916 error_at (location
, "%qE requires a vector base and a scalar"
1920 error_at (location
, "%qE does not support 32-bit vector type %qT",
1921 fndecl
, get_vector_type (type
));
1925 /* Check for some form of vector type, without naming any in particular
1926 as being expected. */
1927 type_suffix_index displacement_type
= infer_vector_type (argno
);
1928 if (displacement_type
== NUM_TYPE_SUFFIXES
)
1931 /* If the displacement type is consistent with the data vector type,
1932 try to find the associated mode suffix. This will fall through
1933 for non-integral displacement types. */
1934 if (type_suffixes
[displacement_type
].element_bits
== required_bits
)
1936 vector_type_index displacement_vector_type
1937 = type_suffixes
[displacement_type
].vector_type
;
1938 mode_suffix_index mode
= find_mode_suffix (NUM_VECTOR_TYPES
,
1939 displacement_vector_type
,
1940 displacement_units ());
1941 if (mode
!= MODE_none
)
1943 if (mode
== MODE_s32offset
1944 && !lookup_form (mode
, type
)
1945 && lookup_form (MODE_u32offset
, type
))
1947 if (type_suffix_ids
[0] == NUM_TYPE_SUFFIXES
)
1948 error_at (location
, "%qE does not support 32-bit sign-extended"
1949 " offsets", fndecl
);
1951 error_at (location
, "%qE does not support sign-extended"
1952 " offsets", fndecl
);
1959 if (type_suffix_ids
[0] == NUM_TYPE_SUFFIXES
)
1961 /* TYPE has been inferred rather than specified by the user,
1962 so mention it in the error messages. */
1964 error_at (location
, "passing %qT to argument %d of %qE, which when"
1965 " loading %qT expects a vector of %d-bit integers",
1966 get_argument_type (argno
), argno
+ 1, fndecl
,
1967 get_vector_type (type
), required_bits
);
1969 error_at (location
, "passing %qT to argument %d of %qE, which when"
1970 " storing %qT expects a vector of %d-bit integers",
1971 get_argument_type (argno
), argno
+ 1, fndecl
,
1972 get_vector_type (type
), required_bits
);
1975 /* TYPE is part of the function name. */
1976 error_at (location
, "passing %qT to argument %d of %qE, which"
1977 " expects a vector of %d-bit integers",
1978 get_argument_type (argno
), argno
+ 1, fndecl
, required_bits
);
1982 /* Require the arguments starting at ARGNO to form a gather-style address.
1983 There are three possible uses:
1985 - for loading into elements of type TYPE (when LOAD_P is true)
1986 - for storing from elements of type TYPE (when LOAD_P is false)
1987 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES)
1989 The three possible addresses are:
1991 - a vector base with no displacement
1992 - a vector base and a scalar displacement
1993 - a scalar (pointer) base and a vector displacement
1995 The overloaded function's mode suffix determines whether there is
1996 a displacement, and if so, what units it uses:
1998 - MODE_none: no displacement
1999 - MODE_offset: the displacement is measured in bytes
2000 - MODE_index: the displacement is measured in elements
2002 Return the mode of the non-overloaded function on success, otherwise
2003 report an error and return MODE_none. */
2005 function_resolver::resolve_gather_address (unsigned int argno
,
2006 type_suffix_index type
,
2009 tree actual
= get_argument_type (argno
);
2010 if (actual
== error_mark_node
)
2013 if (displacement_units () != UNITS_none
)
2015 /* Some form of displacement is needed. First handle a scalar
2016 pointer base and a vector displacement. */
2017 if (scalar_argument_p (argno
))
2018 /* Don't check the pointer type here, since there's only one valid
2019 choice. Leave that to the frontend. */
2020 return resolve_sv_displacement (argno
+ 1, type
, load_p
);
2022 if (!VECTOR_TYPE_P (actual
))
2024 error_at (location
, "passing %qT to argument %d of %qE,"
2025 " which expects a vector or pointer base address",
2026 actual
, argno
+ 1, fndecl
);
2031 /* Check for the correct choice of vector base type. */
2032 vector_type_index base_vector_type
;
2033 if (type
== NUM_TYPE_SUFFIXES
)
2035 /* Since prefetches have no type suffix, there is a free choice
2036 between 32-bit and 64-bit base addresses. */
2037 base_vector_type
= infer_vector_base_type (argno
);
2038 if (base_vector_type
== NUM_VECTOR_TYPES
)
2043 /* Check for some form of vector type, without saying which type
2045 type_suffix_index base_type
= infer_vector_type (argno
);
2046 if (base_type
== NUM_TYPE_SUFFIXES
)
2049 /* Check whether the type is the right one. */
2050 unsigned int required_bits
= type_suffixes
[type
].element_bits
;
2051 gcc_assert (required_bits
== 32 || required_bits
== 64);
2052 type_suffix_index required_type
= (required_bits
== 32
2055 if (required_type
!= base_type
)
2057 error_at (location
, "passing %qT to argument %d of %qE,"
2058 " which expects %qT", actual
, argno
+ 1, fndecl
,
2059 get_vector_type (required_type
));
2062 base_vector_type
= type_suffixes
[base_type
].vector_type
;
2065 /* Check the scalar displacement, if any. */
2066 if (displacement_units () != UNITS_none
2067 && !require_scalar_type (argno
+ 1, "int64_t"))
2070 /* Find the appropriate mode suffix. The checks above should have
2071 weeded out all erroneous cases. */
2072 for (unsigned int mode_i
= 0; mode_i
< ARRAY_SIZE (mode_suffixes
); ++mode_i
)
2074 const mode_suffix_info
&mode
= mode_suffixes
[mode_i
];
2075 if (mode
.base_vector_type
== base_vector_type
2076 && mode
.displacement_vector_type
== NUM_VECTOR_TYPES
2077 && mode
.displacement_units
== displacement_units ())
2078 return mode_suffix_index (mode_i
);
2084 /* Require arguments ARGNO and ARGNO + 1 to form an ADR-style address,
2085 i.e. one with a vector of base addresses and a vector of displacements.
2086 The overloaded function's mode suffix determines the units of the
2087 displacement (bytes for "_offset", elements for "_index").
2089 Return the associated mode suffix on success, otherwise report
2090 an error and return MODE_none. */
2092 function_resolver::resolve_adr_address (unsigned int argno
)
2094 vector_type_index base_type
= infer_vector_base_type (argno
);
2095 if (base_type
== NUM_VECTOR_TYPES
)
2098 vector_type_index displacement_type
2099 = infer_vector_displacement_type (argno
+ 1);
2100 if (displacement_type
== NUM_VECTOR_TYPES
)
2103 mode_suffix_index mode
= find_mode_suffix (base_type
, displacement_type
,
2104 displacement_units ());
2105 if (mode
== MODE_none
)
2107 if (mode_suffix_id
== MODE_offset
)
2108 error_at (location
, "cannot combine a base of type %qT with"
2109 " an offset of type %qT",
2110 get_argument_type (argno
), get_argument_type (argno
+ 1));
2112 error_at (location
, "cannot combine a base of type %qT with"
2113 " an index of type %qT",
2114 get_argument_type (argno
), get_argument_type (argno
+ 1));
2119 /* Require the function to have exactly EXPECTED arguments. Return true
2120 if it does, otherwise report an appropriate error. */
2122 function_resolver::check_num_arguments (unsigned int expected
)
2124 if (m_arglist
.length () < expected
)
2125 error_at (location
, "too few arguments to function %qE", fndecl
);
2126 else if (m_arglist
.length () > expected
)
2127 error_at (location
, "too many arguments to function %qE", fndecl
);
2128 return m_arglist
.length () == expected
;
2131 /* If the function is predicated, check that the first argument is a
2132 suitable governing predicate. Also check that there are NOPS further
2133 arguments after any governing predicate, but don't check what they are.
2135 Return true on success, otherwise report a suitable error.
2136 When returning true:
2138 - set I to the number of the first unchecked argument.
2139 - set NARGS to the total number of arguments. */
2141 function_resolver::check_gp_argument (unsigned int nops
,
2142 unsigned int &i
, unsigned int &nargs
)
2145 if (pred
!= PRED_none
)
2147 /* Unary merge operations should use resolve_unary instead. */
2148 gcc_assert (nops
!= 1 || pred
!= PRED_m
);
2150 if (!check_num_arguments (nargs
)
2151 || !require_vector_type (i
, VECTOR_TYPE_svbool_t
))
2158 if (!check_num_arguments (nargs
))
2165 /* Finish resolving a function whose final argument can be a vector
2166 or a scalar, with the function having an implicit "_n" suffix
2167 in the latter case. This "_n" form might only exist for certain
2170 ARGNO is the index of the final argument. The inferred type suffix
2171 was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE.
2172 EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties
2173 of the final vector or scalar argument, in the same way as for
2174 require_derived_vector_type. INFERRED_TYPE is the inferred type
2175 suffix itself, or NUM_TYPE_SUFFIXES if it's the same as FIRST_TYPE.
2177 Return the function decl of the resolved function on success,
2178 otherwise report a suitable error and return error_mark_node. */
2179 tree
function_resolver::
2180 finish_opt_n_resolution (unsigned int argno
, unsigned int first_argno
,
2181 type_suffix_index first_type
,
2182 type_class_index expected_tclass
,
2183 unsigned int expected_bits
,
2184 type_suffix_index inferred_type
)
2186 if (inferred_type
== NUM_TYPE_SUFFIXES
)
2187 inferred_type
= first_type
;
2188 tree scalar_form
= lookup_form (MODE_n
, inferred_type
);
2190 /* Allow the final argument to be scalar, if an _n form exists. */
2191 if (scalar_argument_p (argno
))
2196 /* Check the vector form normally. If that succeeds, raise an
2197 error about having no corresponding _n form. */
2198 tree res
= resolve_to (mode_suffix_id
, inferred_type
);
2199 if (res
!= error_mark_node
)
2200 error_at (location
, "passing %qT to argument %d of %qE, but its"
2201 " %qT form does not accept scalars",
2202 get_argument_type (argno
), argno
+ 1, fndecl
,
2203 get_vector_type (first_type
));
2204 return error_mark_node
;
2207 /* If an _n form does exist, provide a more accurate message than
2208 require_derived_vector_type would for arguments that are neither
2209 vectors nor scalars. */
2210 if (scalar_form
&& !require_vector_or_scalar_type (argno
))
2211 return error_mark_node
;
2213 /* Check for the correct vector type. */
2214 if (!require_derived_vector_type (argno
, first_argno
, first_type
,
2215 expected_tclass
, expected_bits
))
2216 return error_mark_node
;
2218 return resolve_to (mode_suffix_id
, inferred_type
);
2221 /* Resolve a (possibly predicated) unary function. If the function uses
2222 merge predication or if TREAT_AS_MERGE_P is true, there is an extra
2223 vector argument before the governing predicate that specifies the
2224 values of inactive elements. This argument has the following
2227 - the type class must be the same as for active elements if MERGE_TCLASS
2228 is SAME_TYPE_CLASS, otherwise it must be MERGE_TCLASS itself.
2230 - the element size must be the same as for active elements if MERGE_BITS
2231 is SAME_TYPE_SIZE, otherwise it must be MERGE_BITS itself.
2233 Return the function decl of the resolved function on success,
2234 otherwise report a suitable error and return error_mark_node. */
2236 function_resolver::resolve_unary (type_class_index merge_tclass
,
2237 unsigned int merge_bits
,
2238 bool treat_as_merge_p
)
2240 type_suffix_index type
;
2241 if (pred
== PRED_m
|| treat_as_merge_p
)
2243 if (!check_num_arguments (3))
2244 return error_mark_node
;
2245 if (merge_tclass
== SAME_TYPE_CLASS
&& merge_bits
== SAME_SIZE
)
2247 /* The inactive elements are the same as the active elements,
2248 so we can use normal left-to-right resolution. */
2249 if ((type
= infer_vector_type (0)) == NUM_TYPE_SUFFIXES
2250 || !require_vector_type (1, VECTOR_TYPE_svbool_t
)
2251 || !require_matching_vector_type (2, type
))
2252 return error_mark_node
;
2256 /* The inactive element type is a function of the active one,
2257 so resolve the active one first. */
2258 if (!require_vector_type (1, VECTOR_TYPE_svbool_t
)
2259 || (type
= infer_vector_type (2)) == NUM_TYPE_SUFFIXES
2260 || !require_derived_vector_type (0, 2, type
, merge_tclass
,
2262 return error_mark_node
;
2267 /* We just need to check the predicate (if any) and the single
2269 unsigned int i
, nargs
;
2270 if (!check_gp_argument (1, i
, nargs
)
2271 || (type
= infer_vector_type (i
)) == NUM_TYPE_SUFFIXES
)
2272 return error_mark_node
;
2275 /* Handle convert-like functions in which the first type suffix is
2277 if (type_suffix_ids
[0] != NUM_TYPE_SUFFIXES
)
2278 return resolve_to (mode_suffix_id
, type_suffix_ids
[0], type
);
2280 return resolve_to (mode_suffix_id
, type
);
2283 /* Resolve a (possibly predicated) function that takes NOPS like-typed
2284 vector arguments followed by NIMM integer immediates. Return the
2285 function decl of the resolved function on success, otherwise report
2286 a suitable error and return error_mark_node. */
2288 function_resolver::resolve_uniform (unsigned int nops
, unsigned int nimm
)
2290 unsigned int i
, nargs
;
2291 type_suffix_index type
;
2292 if (!check_gp_argument (nops
+ nimm
, i
, nargs
)
2293 || (type
= infer_vector_type (i
)) == NUM_TYPE_SUFFIXES
)
2294 return error_mark_node
;
2297 for (; i
< nargs
- nimm
; ++i
)
2298 if (!require_matching_vector_type (i
, type
))
2299 return error_mark_node
;
2301 for (; i
< nargs
; ++i
)
2302 if (!require_integer_immediate (i
))
2303 return error_mark_node
;
2305 return resolve_to (mode_suffix_id
, type
);
2308 /* Resolve a (possibly predicated) function that offers a choice between
2311 - NOPS like-typed vector arguments or
2312 - NOPS - 1 like-typed vector arguments followed by a scalar argument
2314 Return the function decl of the resolved function on success,
2315 otherwise report a suitable error and return error_mark_node. */
2317 function_resolver::resolve_uniform_opt_n (unsigned int nops
)
2319 unsigned int i
, nargs
;
2320 type_suffix_index type
;
2321 if (!check_gp_argument (nops
, i
, nargs
)
2322 || (type
= infer_vector_type (i
)) == NUM_TYPE_SUFFIXES
)
2323 return error_mark_node
;
2325 unsigned int first_arg
= i
++;
2326 for (; i
< nargs
- 1; ++i
)
2327 if (!require_matching_vector_type (i
, type
))
2328 return error_mark_node
;
2330 return finish_opt_n_resolution (i
, first_arg
, type
);
2333 /* If the call is erroneous, report an appropriate error and return
2334 error_mark_node. Otherwise, if the function is overloaded, return
2335 the decl of the non-overloaded function. Return NULL_TREE otherwise,
2336 indicating that the call should be processed in the normal way. */
2338 function_resolver::resolve ()
2340 return shape
->resolve (*this);
2343 function_checker::function_checker (location_t location
,
2344 const function_instance
&instance
,
2345 tree fndecl
, tree fntype
,
2346 unsigned int nargs
, tree
*args
)
2347 : function_call_info (location
, instance
, fndecl
),
2348 m_fntype (fntype
), m_nargs (nargs
), m_args (args
),
2349 /* We don't have to worry about unary _m operations here, since they
2350 never have arguments that need checking. */
2351 m_base_arg (pred
!= PRED_none
? 1 : 0)
2355 /* Return true if argument ARGNO exists. which it might not for
2356 erroneous calls. It is safe to wave through checks if this
2357 function returns false. */
2359 function_checker::argument_exists_p (unsigned int argno
)
2361 gcc_assert (argno
< (unsigned int) type_num_arguments (m_fntype
));
2362 return argno
< m_nargs
;
2365 /* Check that argument ARGNO is an integer constant expression and
2366 store its value in VALUE_OUT if so. The caller should first
2367 check that argument ARGNO exists. */
2369 function_checker::require_immediate (unsigned int argno
,
2370 HOST_WIDE_INT
&value_out
)
2372 gcc_assert (argno
< m_nargs
);
2373 tree arg
= m_args
[argno
];
2375 /* The type and range are unsigned, so read the argument as an
2376 unsigned rather than signed HWI. */
2377 if (!tree_fits_uhwi_p (arg
))
2379 report_non_ice (location
, fndecl
, argno
);
2383 /* ...but treat VALUE_OUT as signed for error reporting, since printing
2384 -1 is more user-friendly than the maximum uint64_t value. */
2385 value_out
= tree_to_uhwi (arg
);
2389 /* Check that argument REL_ARGNO is an integer constant expression that
2390 has the value VALUE0 or VALUE1. REL_ARGNO counts from the end of the
2391 predication arguments. */
2393 function_checker::require_immediate_either_or (unsigned int rel_argno
,
2394 HOST_WIDE_INT value0
,
2395 HOST_WIDE_INT value1
)
2397 unsigned int argno
= m_base_arg
+ rel_argno
;
2398 if (!argument_exists_p (argno
))
2401 HOST_WIDE_INT actual
;
2402 if (!require_immediate (argno
, actual
))
2405 if (actual
!= value0
&& actual
!= value1
)
2407 report_neither_nor (location
, fndecl
, argno
, actual
, 90, 270);
2414 /* Check that argument REL_ARGNO is an integer constant expression that has
2415 a valid value for enumeration type TYPE. REL_ARGNO counts from the end
2416 of the predication arguments. */
2418 function_checker::require_immediate_enum (unsigned int rel_argno
, tree type
)
2420 unsigned int argno
= m_base_arg
+ rel_argno
;
2421 if (!argument_exists_p (argno
))
2424 HOST_WIDE_INT actual
;
2425 if (!require_immediate (argno
, actual
))
2428 for (tree entry
= TYPE_VALUES (type
); entry
; entry
= TREE_CHAIN (entry
))
2430 /* The value is an INTEGER_CST for C and a CONST_DECL wrapper
2431 around an INTEGER_CST for C++. */
2432 tree value
= TREE_VALUE (entry
);
2433 if (TREE_CODE (value
) == CONST_DECL
)
2434 value
= DECL_INITIAL (value
);
2435 if (wi::to_widest (value
) == actual
)
2439 report_not_enum (location
, fndecl
, argno
, actual
, type
);
2443 /* Check that argument REL_ARGNO is suitable for indexing argument
2444 REL_ARGNO - 1, in groups of GROUP_SIZE elements. REL_ARGNO counts
2445 from the end of the predication arguments. */
2447 function_checker::require_immediate_lane_index (unsigned int rel_argno
,
2448 unsigned int group_size
)
2450 unsigned int argno
= m_base_arg
+ rel_argno
;
2451 if (!argument_exists_p (argno
))
2454 /* Get the type of the previous argument. tree_argument_type wants a
2455 1-based number, whereas ARGNO is 0-based. */
2456 machine_mode mode
= TYPE_MODE (type_argument_type (m_fntype
, argno
));
2457 gcc_assert (VECTOR_MODE_P (mode
));
2458 unsigned int nlanes
= 128 / (group_size
* GET_MODE_UNIT_BITSIZE (mode
));
2459 return require_immediate_range (rel_argno
, 0, nlanes
- 1);
2462 /* Check that argument REL_ARGNO is an integer constant expression that
2463 has one of the given values. */
2465 function_checker::require_immediate_one_of (unsigned int rel_argno
,
2466 HOST_WIDE_INT value0
,
2467 HOST_WIDE_INT value1
,
2468 HOST_WIDE_INT value2
,
2469 HOST_WIDE_INT value3
)
2471 unsigned int argno
= m_base_arg
+ rel_argno
;
2472 if (!argument_exists_p (argno
))
2475 HOST_WIDE_INT actual
;
2476 if (!require_immediate (argno
, actual
))
2479 if (actual
!= value0
2482 && actual
!= value3
)
2484 report_not_one_of (location
, fndecl
, argno
, actual
,
2485 value0
, value1
, value2
, value3
);
2492 /* Check that argument REL_ARGNO is an integer constant expression in the
2493 range [MIN, MAX]. REL_ARGNO counts from the end of the predication
2496 function_checker::require_immediate_range (unsigned int rel_argno
,
2500 unsigned int argno
= m_base_arg
+ rel_argno
;
2501 if (!argument_exists_p (argno
))
2504 /* Required because of the tree_to_uhwi -> HOST_WIDE_INT conversion
2505 in require_immediate. */
2506 gcc_assert (min
>= 0 && min
<= max
);
2507 HOST_WIDE_INT actual
;
2508 if (!require_immediate (argno
, actual
))
2511 if (!IN_RANGE (actual
, min
, max
))
2513 report_out_of_range (location
, fndecl
, argno
, actual
, min
, max
);
2520 /* Perform semantic checks on the call. Return true if the call is valid,
2521 otherwise report a suitable error. */
2523 function_checker::check ()
2525 function_args_iterator iter
;
2528 FOREACH_FUNCTION_ARGS (m_fntype
, type
, iter
)
2530 if (type
== void_type_node
|| i
>= m_nargs
)
2534 && TREE_CODE (type
) == ENUMERAL_TYPE
2535 && !require_immediate_enum (i
- m_base_arg
, type
))
2541 return shape
->check (*this);
2544 gimple_folder::gimple_folder (const function_instance
&instance
, tree fndecl
,
2545 gimple_stmt_iterator
*gsi_in
, gcall
*call_in
)
2546 : function_call_info (gimple_location (call_in
), instance
, fndecl
),
2547 gsi (gsi_in
), call (call_in
), lhs (gimple_call_lhs (call_in
))
2551 /* VALUE might be a vector of type VECTYPE or a single scalar element.
2552 Duplicate it into a vector of type VECTYPE in the latter case, adding any
2553 new statements to STMTS. */
2555 gimple_folder::force_vector (gimple_seq
&stmts
, tree vectype
, tree value
)
2557 if (!VECTOR_TYPE_P (TREE_TYPE (value
)))
2558 value
= gimple_build_vector_from_val (&stmts
, vectype
, value
);
2562 /* Convert predicate argument ARGNO so that it has the type appropriate for
2563 an operation on VECTYPE. Add any new statements to STMTS. */
2565 gimple_folder::convert_pred (gimple_seq
&stmts
, tree vectype
,
2568 tree pred
= gimple_call_arg (call
, argno
);
2569 if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred
)),
2570 TYPE_VECTOR_SUBPARTS (vectype
)))
2573 return gimple_build (&stmts
, VIEW_CONVERT_EXPR
,
2574 truth_type_for (vectype
), pred
);
2577 /* Return a pointer to the address in a contiguous load or store,
2578 given that each memory vector has type VECTYPE. Add any new
2579 statements to STMTS. */
2581 gimple_folder::fold_contiguous_base (gimple_seq
&stmts
, tree vectype
)
2583 tree base
= gimple_call_arg (call
, 1);
2584 if (mode_suffix_id
== MODE_vnum
)
2586 tree offset
= gimple_call_arg (call
, 2);
2587 offset
= gimple_convert (&stmts
, sizetype
, offset
);
2588 offset
= gimple_build (&stmts
, MULT_EXPR
, sizetype
, offset
,
2589 TYPE_SIZE_UNIT (vectype
));
2590 base
= gimple_build (&stmts
, POINTER_PLUS_EXPR
, TREE_TYPE (base
),
2596 /* Return the alignment and TBAA argument to an internal load or store
2597 function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses
2598 memory elements of type TYPE. */
2600 gimple_folder::load_store_cookie (tree type
)
2602 return build_int_cst (build_pointer_type (type
), TYPE_ALIGN (type
));
2605 /* Fold the call to a call to INSTANCE, with the same arguments. */
2607 gimple_folder::redirect_call (const function_instance
&instance
)
2609 registered_function
*rfn
2610 = function_table
->find_with_hash (instance
, instance
.hash ());
2614 gimple_call_set_fndecl (call
, rfn
->decl
);
2618 /* Fold the call to a PTRUE, taking the element size from type suffix 0. */
2620 gimple_folder::fold_to_ptrue ()
2622 tree svbool_type
= TREE_TYPE (lhs
);
2623 tree bool_type
= TREE_TYPE (svbool_type
);
2624 unsigned int element_bytes
= type_suffix (0).element_bytes
;
2626 /* The return type is svbool_t for all type suffixes, thus for b8 we
2627 want { 1, 1, 1, 1, ... }, for b16 we want { 1, 0, 1, 0, ... }, etc. */
2628 tree_vector_builder
builder (svbool_type
, element_bytes
, 1);
2629 builder
.quick_push (build_all_ones_cst (bool_type
));
2630 for (unsigned int i
= 1; i
< element_bytes
; ++i
)
2631 builder
.quick_push (build_zero_cst (bool_type
));
2632 return gimple_build_assign (lhs
, builder
.build ());
2635 /* Fold the call to a PFALSE. */
2637 gimple_folder::fold_to_pfalse ()
2639 return gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2642 /* Fold an operation to a constant predicate in which the first VL
2643 elements are set and the rest are clear. Take the element size
2644 from type suffix 0. */
2646 gimple_folder::fold_to_vl_pred (unsigned int vl
)
2648 tree vectype
= TREE_TYPE (lhs
);
2649 tree element_type
= TREE_TYPE (vectype
);
2650 tree minus_one
= build_all_ones_cst (element_type
);
2651 tree zero
= build_zero_cst (element_type
);
2652 unsigned int element_bytes
= type_suffix (0).element_bytes
;
2654 /* Construct COUNT elements that contain the ptrue followed by
2655 a repeating sequence of COUNT elements. */
2656 unsigned int count
= constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype
));
2657 gcc_assert (vl
* element_bytes
<= count
);
2658 tree_vector_builder
builder (vectype
, count
, 2);
2659 for (unsigned int i
= 0; i
< count
* 2; ++i
)
2661 bool bit
= (i
& (element_bytes
- 1)) == 0 && i
< vl
* element_bytes
;
2662 builder
.quick_push (bit
? minus_one
: zero
);
2664 return gimple_build_assign (lhs
, builder
.build ());
2667 /* Try to fold the call. Return the new statement on success and null
2670 gimple_folder::fold ()
2672 /* Don't fold anything when SVE is disabled; emit an error during
2673 expansion instead. */
2677 /* Punt if the function has a return type and no result location is
2678 provided. The attributes should allow target-independent code to
2679 remove the calls if appropriate. */
2680 if (!lhs
&& TREE_TYPE (gimple_call_fntype (call
)) != void_type_node
)
2683 return base
->fold (*this);
2686 function_expander::function_expander (const function_instance
&instance
,
2687 tree fndecl
, tree call_expr_in
,
2688 rtx possible_target_in
)
2689 : function_call_info (EXPR_LOCATION (call_expr_in
), instance
, fndecl
),
2690 call_expr (call_expr_in
), possible_target (possible_target_in
)
2694 /* Return the handler of direct optab OP for type suffix SUFFIX_I. */
2696 function_expander::direct_optab_handler (optab op
, unsigned int suffix_i
)
2698 return ::direct_optab_handler (op
, vector_mode (suffix_i
));
2701 /* Choose between signed and unsigned direct optabs SIGNED_OP and
2702 UNSIGNED_OP based on the signedness of type suffix SUFFIX_I, then
2703 pick the appropriate optab handler for the mode. Use MODE as the
2704 mode if given, otherwise use the mode of type suffix SUFFIX_I. */
2706 function_expander::direct_optab_handler_for_sign (optab signed_op
,
2708 unsigned int suffix_i
,
2711 if (mode
== VOIDmode
)
2712 mode
= vector_mode (suffix_i
);
2713 optab op
= type_suffix (suffix_i
).unsigned_p
? unsigned_op
: signed_op
;
2714 return ::direct_optab_handler (op
, mode
);
2717 /* Return true if X overlaps any input. */
2719 function_expander::overlaps_input_p (rtx x
)
2721 for (unsigned int i
= 0; i
< args
.length (); ++i
)
2722 if (reg_overlap_mentioned_p (x
, args
[i
]))
2727 /* Convert ptr_mode value X to Pmode. */
2729 function_expander::convert_to_pmode (rtx x
)
2731 if (ptr_mode
== SImode
)
2732 x
= simplify_gen_unary (ZERO_EXTEND
, DImode
, x
, SImode
);
2736 /* Return the base address for a contiguous load or store function.
2737 MEM_MODE is the mode of the addressed memory. */
2739 function_expander::get_contiguous_base (machine_mode mem_mode
)
2741 rtx base
= convert_to_pmode (args
[1]);
2742 if (mode_suffix_id
== MODE_vnum
)
2744 /* Use the size of the memory mode for extending loads and truncating
2745 stores. Use the size of a full vector for non-extending loads
2746 and non-truncating stores (including svld[234] and svst[234]). */
2747 poly_int64 size
= ordered_min (GET_MODE_SIZE (mem_mode
),
2748 BYTES_PER_SVE_VECTOR
);
2749 rtx offset
= gen_int_mode (size
, Pmode
);
2750 offset
= simplify_gen_binary (MULT
, Pmode
, args
[2], offset
);
2751 base
= simplify_gen_binary (PLUS
, Pmode
, base
, offset
);
2756 /* For a function that does the equivalent of:
2758 OUTPUT = COND ? FN (INPUTS) : FALLBACK;
2760 return the value of FALLBACK.
2762 MODE is the mode of OUTPUT. NOPS is the number of operands in INPUTS.
2763 MERGE_ARGNO is the argument that provides FALLBACK for _m functions,
2764 or DEFAULT_MERGE_ARGNO if we should apply the usual rules.
2766 ARGNO is the caller's index into args. If the returned value is
2767 argument 0 (as for unary _m operations), increment ARGNO past the
2768 returned argument. */
2770 function_expander::get_fallback_value (machine_mode mode
, unsigned int nops
,
2771 unsigned int merge_argno
,
2772 unsigned int &argno
)
2775 return CONST0_RTX (mode
);
2777 gcc_assert (pred
== PRED_m
|| pred
== PRED_x
);
2778 if (merge_argno
== DEFAULT_MERGE_ARGNO
)
2779 merge_argno
= nops
== 1 && pred
== PRED_m
? 0 : 1;
2781 if (merge_argno
== 0)
2782 return args
[argno
++];
2784 return args
[merge_argno
];
2787 /* Return a REG rtx that can be used for the result of the function,
2788 using the preferred target if suitable. */
2790 function_expander::get_reg_target ()
2792 machine_mode target_mode
= TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
)));
2793 if (!possible_target
|| GET_MODE (possible_target
) != target_mode
)
2794 possible_target
= gen_reg_rtx (target_mode
);
2795 return possible_target
;
2798 /* As for get_reg_target, but make sure that the returned REG does not
2799 overlap any inputs. */
2801 function_expander::get_nonoverlapping_reg_target ()
2803 if (possible_target
&& overlaps_input_p (possible_target
))
2804 possible_target
= NULL_RTX
;
2805 return get_reg_target ();
2808 /* Add an output operand to the instruction we're building, which has
2809 code ICODE. Bind the output to the preferred target rtx if possible. */
2811 function_expander::add_output_operand (insn_code icode
)
2813 unsigned int opno
= m_ops
.length ();
2814 machine_mode mode
= insn_data
[icode
].operand
[opno
].mode
;
2815 m_ops
.safe_grow (opno
+ 1, true);
2816 create_output_operand (&m_ops
.last (), possible_target
, mode
);
2819 /* Add an input operand to the instruction we're building, which has
2820 code ICODE. Calculate the value of the operand as follows:
2822 - If the operand is a vector and X is not, broadcast X to fill a
2823 vector of the appropriate mode.
2825 - Otherwise, if the operand is a predicate, coerce X to have the
2826 mode that the instruction expects. In this case X is known to be
2827 VNx16BImode (the mode of svbool_t).
2829 - Otherwise use X directly. The expand machinery checks that X has
2830 the right mode for the instruction. */
2832 function_expander::add_input_operand (insn_code icode
, rtx x
)
2834 unsigned int opno
= m_ops
.length ();
2835 const insn_operand_data
&operand
= insn_data
[icode
].operand
[opno
];
2836 machine_mode mode
= operand
.mode
;
2837 if (mode
== VOIDmode
)
2839 /* The only allowable use of VOIDmode is the wildcard
2840 aarch64_any_register_operand, which is used to avoid
2841 combinatorial explosion in the reinterpret patterns. */
2842 gcc_assert (operand
.predicate
== aarch64_any_register_operand
);
2843 mode
= GET_MODE (x
);
2845 else if (!VECTOR_MODE_P (GET_MODE (x
)) && VECTOR_MODE_P (mode
))
2846 x
= expand_vector_broadcast (mode
, x
);
2847 else if (GET_MODE_CLASS (mode
) == MODE_VECTOR_BOOL
)
2849 gcc_assert (GET_MODE (x
) == VNx16BImode
);
2850 x
= gen_lowpart (mode
, x
);
2852 m_ops
.safe_grow (m_ops
.length () + 1, true);
2853 create_input_operand (&m_ops
.last (), x
, mode
);
2856 /* Add an integer operand with value X to the instruction. */
2858 function_expander::add_integer_operand (HOST_WIDE_INT x
)
2860 m_ops
.safe_grow (m_ops
.length () + 1, true);
2861 create_integer_operand (&m_ops
.last (), x
);
2864 /* Add a memory operand with mode MODE and address ADDR. */
2866 function_expander::add_mem_operand (machine_mode mode
, rtx addr
)
2868 /* Exception for OImode for the ld1ro intrinsics.
2869 They act on 256 bit octaword data, and it's just easier to use a scalar
2870 mode to represent that than add a new vector mode solely for the purpose
2871 of this intrinsic. */
2872 gcc_assert (VECTOR_MODE_P (mode
) || mode
== OImode
);
2873 rtx mem
= gen_rtx_MEM (mode
, memory_address (mode
, addr
));
2874 /* The memory is only guaranteed to be element-aligned. */
2875 set_mem_align (mem
, GET_MODE_ALIGNMENT (GET_MODE_INNER (mode
)));
2876 add_fixed_operand (mem
);
2879 /* Add an address operand with value X. The static operand data says
2880 what mode and form the address must have. */
2882 function_expander::add_address_operand (rtx x
)
2884 m_ops
.safe_grow (m_ops
.length () + 1, true);
2885 create_address_operand (&m_ops
.last (), x
);
2888 /* Add an operand that must be X. The only way of legitimizing an
2889 invalid X is to reload the address of a MEM. */
2891 function_expander::add_fixed_operand (rtx x
)
2893 m_ops
.safe_grow (m_ops
.length () + 1, true);
2894 create_fixed_operand (&m_ops
.last (), x
);
2897 /* Generate instruction ICODE, given that its operands have already
2898 been added to M_OPS. Return the value of the first operand. */
2900 function_expander::generate_insn (insn_code icode
)
2902 expand_insn (icode
, m_ops
.length (), m_ops
.address ());
2903 return function_returns_void_p () ? const0_rtx
: m_ops
[0].value
;
2906 /* Convert the arguments to a gather/scatter function into the
2907 associated md operands. Argument ARGNO is the scalar or vector base and
2908 argument ARGNO + 1 is the scalar or vector displacement (if applicable).
2909 The md pattern expects:
2912 - a vector displacement
2914 If SCALED_P is true, it also expects:
2916 - a const_int that is 1 if the displacement is zero-extended from 32 bits
2917 - a scaling multiplier (1 for bytes, 2 for .h indices, etc.).
2919 If SCALED_P is false, the displacement is implicitly zero-extended
2920 and the scaling multiplier is implicitly 1. */
2922 function_expander::prepare_gather_address_operands (unsigned int argno
,
2925 machine_mode mem_mode
= memory_vector_mode ();
2926 tree vector_type
= base_vector_type ();
2927 units_index units
= displacement_units ();
2929 if (units
== UNITS_none
)
2931 /* Vector base, no displacement. Convert to an integer zero base
2932 and a vector byte offset. */
2933 args
.quick_insert (argno
, const0_rtx
);
2934 units
= UNITS_bytes
;
2936 else if (vector_type
)
2938 /* Vector base, scalar displacement. Convert to a scalar base and
2939 a vector byte offset. */
2940 std::swap (args
[argno
], args
[argno
+ 1]);
2941 if (units
== UNITS_elements
)
2946 /* Scalar base, vector displacement. This is the order that the md
2948 args
[argno
] = convert_to_pmode (args
[argno
]);
2949 vector_type
= displacement_vector_type ();
2950 if (units
== UNITS_elements
&& !scaled_p
)
2951 shift_idx
= argno
+ 1;
2953 tree scalar_displacement_type
= TREE_TYPE (vector_type
);
2957 machine_mode arg_mode
= GET_MODE (args
[shift_idx
]);
2958 if (arg_mode
== VOIDmode
)
2960 unsigned int elt_bytes
= GET_MODE_UNIT_SIZE (mem_mode
);
2961 rtx shift
= gen_int_mode (exact_log2 (elt_bytes
), DImode
);
2962 args
[shift_idx
] = simplify_gen_binary (ASHIFT
, arg_mode
,
2963 args
[shift_idx
], shift
);
2964 units
= UNITS_bytes
;
2967 bool uxtw_p
= (TYPE_PRECISION (scalar_displacement_type
) == 64
2968 || TYPE_UNSIGNED (scalar_displacement_type
));
2969 unsigned int scale
= (units
== UNITS_bytes
2970 ? 1 : GET_MODE_UNIT_SIZE (mem_mode
));
2974 args
.quick_insert (argno
+ 2, GEN_INT (uxtw_p
));
2975 args
.quick_insert (argno
+ 3, GEN_INT (scale
));
2978 gcc_assert (uxtw_p
&& scale
== 1);
2981 /* The final argument is an immediate svprfop value. Add two fake arguments
2982 to represent the rw and locality operands of a PREFETCH rtx. */
2984 function_expander::prepare_prefetch_operands ()
2986 unsigned int prfop
= INTVAL (args
.last ());
2987 /* Bit 3 of the prfop selects stores over loads. */
2988 args
.quick_push (GEN_INT ((prfop
& 8) != 0));
2989 /* Bits 1 and 2 specify the locality; 0-based for svprfop but
2990 1-based for PREFETCH. */
2991 args
.quick_push (GEN_INT (((prfop
>> 1) & 3) + 1));
2994 /* Add a dummy argument to indicate whether predicate argument ARGNO
2995 is all-true when interpreted in mode PRED_MODE. The hint goes
2996 immediately after ARGNO. */
2998 function_expander::add_ptrue_hint (unsigned int argno
, machine_mode pred_mode
)
3000 rtx pred
= gen_lowpart (pred_mode
, args
[argno
]);
3001 int hint
= (pred
== CONSTM1_RTX (pred_mode
)
3002 ? SVE_KNOWN_PTRUE
: SVE_MAYBE_NOT_PTRUE
);
3003 args
.quick_insert (argno
+ 1, gen_int_mode (hint
, SImode
));
3006 /* Rotate inputs args[START:END] one position to the left, so that
3007 args[START] becomes args[END - 1]. */
3009 function_expander::rotate_inputs_left (unsigned int start
, unsigned int end
)
3011 rtx new_last
= args
[start
];
3012 for (unsigned int i
= start
; i
< end
- 1; ++i
)
3013 args
[i
] = args
[i
+ 1];
3014 args
[end
- 1] = new_last
;
3017 /* Return true if the negation of argument ARGNO can be folded away,
3018 replacing it with the negated value if so. MODE is the associated
3019 vector mode, but the argument could be a single element. The main
3020 case this handles is constant arguments. */
3022 function_expander::try_negating_argument (unsigned int argno
,
3025 rtx x
= args
[argno
];
3026 if (!VECTOR_MODE_P (GET_MODE (x
)))
3027 mode
= GET_MODE_INNER (mode
);
3029 x
= simplify_unary_operation (NEG
, mode
, x
, mode
);
3037 /* Implement the call using instruction ICODE, with a 1:1 mapping between
3038 arguments and input operands. */
3040 function_expander::use_exact_insn (insn_code icode
)
3042 unsigned int nops
= insn_data
[icode
].n_operands
;
3043 if (!function_returns_void_p ())
3045 add_output_operand (icode
);
3048 for (unsigned int i
= 0; i
< nops
; ++i
)
3049 add_input_operand (icode
, args
[i
]);
3050 return generate_insn (icode
);
3053 /* Implement the call using instruction ICODE, which does not use a
3054 governing predicate. We must therefore drop the GP from an _x call. */
3056 function_expander::use_unpred_insn (insn_code icode
)
3058 /* We can't drop the predicate for _z and _m. */
3059 gcc_assert (pred
== PRED_x
|| pred
== PRED_none
);
3060 /* Discount the output operand. */
3061 unsigned int nops
= insn_data
[icode
].n_operands
- 1;
3062 /* Drop the predicate argument in the case of _x predication. */
3063 unsigned int bias
= (pred
== PRED_x
? 1 : 0);
3066 add_output_operand (icode
);
3067 for (; i
< nops
; ++i
)
3068 add_input_operand (icode
, args
[i
+ bias
]);
3070 return generate_insn (icode
);
3073 /* Implement the call using instruction ICODE, which is a predicated
3074 operation that returns arbitrary values for inactive lanes. */
3076 function_expander::use_pred_x_insn (insn_code icode
)
3078 /* At present we never need to handle PRED_none, which would involve
3079 creating a new predicate rather than using one supplied by the user. */
3080 gcc_assert (pred
== PRED_x
);
3081 /* Discount the output operand. */
3082 unsigned int nops
= args
.length () - 1;
3084 bool has_float_operand_p
= FLOAT_MODE_P (insn_data
[icode
].operand
[0].mode
);
3086 /* Add the normal operands. */
3087 add_output_operand (icode
);
3088 add_input_operand (icode
, args
[0]);
3089 for (unsigned int i
= 0; i
< nops
; ++i
)
3091 add_input_operand (icode
, args
[i
+ 1]);
3092 if (FLOAT_MODE_P (GET_MODE (args
[i
+ 1])))
3093 has_float_operand_p
= true;
3096 if (has_float_operand_p
)
3098 /* Add a flag that indicates whether unpredicated instructions
3100 rtx pred
= m_ops
[1].value
;
3101 if (flag_trapping_math
&& pred
!= CONST1_RTX (GET_MODE (pred
)))
3102 add_integer_operand (SVE_STRICT_GP
);
3104 add_integer_operand (SVE_RELAXED_GP
);
3107 return generate_insn (icode
);
3110 /* Implement the call using instruction ICODE, which does the equivalent of:
3112 OUTPUT = COND ? FN (INPUTS) : FALLBACK;
3114 The instruction operands are in the order above: OUTPUT, COND, INPUTS
3115 and FALLBACK. MERGE_ARGNO is the argument that provides FALLBACK for _m
3116 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
3118 function_expander::use_cond_insn (insn_code icode
, unsigned int merge_argno
)
3120 /* At present we never need to handle PRED_none, which would involve
3121 creating a new predicate rather than using one supplied by the user. */
3122 gcc_assert (pred
!= PRED_none
);
3123 /* Discount the output, predicate and fallback value. */
3124 unsigned int nops
= insn_data
[icode
].n_operands
- 3;
3125 machine_mode mode
= insn_data
[icode
].operand
[0].mode
;
3127 unsigned int opno
= 0;
3128 rtx fallback_arg
= get_fallback_value (mode
, nops
, merge_argno
, opno
);
3129 rtx pred
= args
[opno
++];
3131 add_output_operand (icode
);
3132 add_input_operand (icode
, pred
);
3133 for (unsigned int i
= 0; i
< nops
; ++i
)
3134 add_input_operand (icode
, args
[opno
+ i
]);
3135 add_input_operand (icode
, fallback_arg
);
3136 return generate_insn (icode
);
3139 /* Implement the call using instruction ICODE, which is a select-like
3140 operation with the following operands:
3147 MERGE_ARGNO is the argument that provides the "false" value for _m
3148 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
3150 function_expander::use_vcond_mask_insn (insn_code icode
,
3151 unsigned int merge_argno
)
3153 machine_mode mode
= vector_mode (0);
3155 unsigned int opno
= 0;
3156 rtx false_arg
= get_fallback_value (mode
, 1, merge_argno
, opno
);
3157 rtx pred_arg
= args
[opno
++];
3158 rtx true_arg
= args
[opno
++];
3160 add_output_operand (icode
);
3161 add_input_operand (icode
, true_arg
);
3162 add_input_operand (icode
, false_arg
);
3163 add_input_operand (icode
, pred_arg
);
3164 return generate_insn (icode
);
3167 /* Implement the call using instruction ICODE, which loads memory operand 1
3168 into register operand 0 under the control of predicate operand 2.
3169 Extending loads have a further predicate (operand 3) that nominally
3170 controls the extension. */
3172 function_expander::use_contiguous_load_insn (insn_code icode
)
3174 machine_mode mem_mode
= memory_vector_mode ();
3176 add_output_operand (icode
);
3177 add_mem_operand (mem_mode
, get_contiguous_base (mem_mode
));
3178 add_input_operand (icode
, args
[0]);
3179 if (GET_MODE_UNIT_BITSIZE (mem_mode
) < type_suffix (0).element_bits
)
3180 add_input_operand (icode
, CONSTM1_RTX (VNx16BImode
));
3181 return generate_insn (icode
);
3184 /* Implement the call using instruction ICODE, which prefetches from
3185 address operand 1 under the control of predicate operand 0.
3186 Operands 2, 3 and 4 respectively specify the svprfop value,
3187 the PREFETCH rw flag and the PREFETCH locality. */
3189 function_expander::use_contiguous_prefetch_insn (insn_code icode
)
3191 add_input_operand (icode
, args
[0]);
3192 add_address_operand (get_contiguous_base (VNx16QImode
));
3193 for (unsigned int i
= args
.length () - 3; i
< args
.length (); ++i
)
3194 add_input_operand (icode
, args
[i
]);
3195 return generate_insn (icode
);
3198 /* Implement the call using instruction ICODE, which stores register operand 1
3199 into memory operand 0 under the control of predicate operand 2. */
3201 function_expander::use_contiguous_store_insn (insn_code icode
)
3203 machine_mode mem_mode
= memory_vector_mode ();
3205 add_mem_operand (mem_mode
, get_contiguous_base (mem_mode
));
3206 add_input_operand (icode
, args
.last ());
3207 add_input_operand (icode
, args
[0]);
3208 return generate_insn (icode
);
3211 /* Implement the call using one of the following strategies, chosen in order:
3213 (1) "aarch64_pred_<optab><mode>_z" for PRED_z predicate functions
3215 (2) "aarch64_pred_<optab><mode>" for PRED_x functions
3217 (3) a normal unpredicated optab for PRED_none and PRED_x functions,
3218 dropping the predicate in the latter case
3220 (4) an unpredicated "aarch64_sve_<code_optab><mode>" for PRED_none and
3221 PRED_x functions, again dropping the predicate for PRED_x
3223 (5) "cond_<optab><mode>" otherwise
3225 where <optab> corresponds to:
3227 - CODE_FOR_SINT for signed integers
3228 - CODE_FOR_UINT for unsigned integers
3229 - UNSPEC_FOR_FP for floating-point values
3231 and where <code_optab> is like <optab>, but uses CODE_FOR_SINT instead
3232 of UNSPEC_FOR_FP for floating-point values.
3234 MERGE_ARGNO is the argument that provides the values of inactive lanes for
3235 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
3237 function_expander::map_to_rtx_codes (rtx_code code_for_sint
,
3238 rtx_code code_for_uint
,
3240 unsigned int merge_argno
)
3242 machine_mode mode
= vector_mode (0);
3243 rtx_code code
= (type_suffix (0).unsigned_p
? code_for_uint
: code_for_sint
);
3246 /* Handle predicate logic operations, which always use _z predication. */
3247 if (type_suffix (0).tclass
== TYPE_bool
)
3249 gcc_assert (pred
== PRED_z
&& code_for_uint
== code_for_sint
);
3250 return use_exact_insn (code_for_aarch64_pred_z (code
, mode
));
3253 /* First try using UNSPEC_PRED_X patterns for _x predication,
3257 if (type_suffix (0).integer_p
)
3258 icode
= maybe_code_for_aarch64_pred (code
, mode
);
3260 icode
= maybe_code_for_aarch64_pred (unspec_for_fp
, mode
);
3261 if (icode
!= CODE_FOR_nothing
)
3262 return use_pred_x_insn (icode
);
3265 /* Otherwise expand PRED_none and PRED_x operations without a predicate.
3266 Floating-point operations conventionally use the signed rtx code. */
3267 if (pred
== PRED_none
|| pred
== PRED_x
)
3269 icode
= direct_optab_handler (code_to_optab (code
), 0);
3270 if (icode
== CODE_FOR_nothing
)
3271 icode
= code_for_aarch64_sve (code
, mode
);
3272 return use_unpred_insn (icode
);
3275 /* Don't use cond_*_optabs here, since not all codes have one yet. */
3276 if (type_suffix (0).integer_p
)
3277 icode
= code_for_cond (code
, mode
);
3279 icode
= code_for_cond (unspec_for_fp
, mode
);
3280 return use_cond_insn (icode
, merge_argno
);
3283 /* Implement the call using one of the following strategies, chosen in order:
3285 (1) "aarch64_pred_<optab><mode>" for PRED_x functions; this is a
3288 (2) "aarch64_sve_<optab><mode>" for PRED_none and PRED_x functions;
3289 this is an unpredicated pattern
3291 (3) "cond_<optab><mode>" otherwise
3293 where <optab> corresponds to:
3295 - UNSPEC_FOR_SINT for signed integers
3296 - UNSPEC_FOR_UINT for unsigned integers
3297 - UNSPEC_FOR_FP for floating-point values
3299 MERGE_ARGNO is the argument that provides the values of inactive lanes for
3300 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
3302 function_expander::map_to_unspecs (int unspec_for_sint
, int unspec_for_uint
,
3303 int unspec_for_fp
, unsigned int merge_argno
)
3305 machine_mode mode
= vector_mode (0);
3306 int unspec
= (!type_suffix (0).integer_p
? unspec_for_fp
3307 : type_suffix (0).unsigned_p
? unspec_for_uint
3312 insn_code icode
= maybe_code_for_aarch64_pred (unspec
, mode
);
3313 if (icode
!= CODE_FOR_nothing
)
3314 return use_pred_x_insn (icode
);
3317 if (pred
== PRED_none
|| pred
== PRED_x
)
3319 insn_code icode
= maybe_code_for_aarch64_sve (unspec
, mode
);
3320 if (icode
!= CODE_FOR_nothing
)
3321 return use_unpred_insn (icode
);
3324 insn_code icode
= code_for_cond (unspec
, vector_mode (0));
3325 return use_cond_insn (icode
, merge_argno
);
3328 /* Expand the call and return its lhs. */
3330 function_expander::expand ()
3332 unsigned int nargs
= call_expr_nargs (call_expr
);
3333 args
.reserve (nargs
);
3334 for (unsigned int i
= 0; i
< nargs
; ++i
)
3335 args
.quick_push (expand_normal (CALL_EXPR_ARG (call_expr
, i
)));
3337 return base
->expand (*this);
3340 /* Register the built-in SVE ABI types, such as __SVBool_t. */
3342 register_builtin_types ()
3344 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
3345 scalar_types[VECTOR_TYPE_ ## ACLE_NAME] = SCALAR_TYPE;
3346 #include "aarch64-sve-builtins.def"
3348 for (unsigned int i
= 0; i
< NUM_VECTOR_TYPES
; ++i
)
3350 tree eltype
= scalar_types
[i
];
3352 unsigned int num_zr
= 0, num_pr
= 0;
3353 if (eltype
== boolean_type_node
)
3355 vectype
= build_truth_vector_type_for_mode (BYTES_PER_SVE_VECTOR
,
3357 gcc_assert (TYPE_MODE (vectype
) == VNx16BImode
3358 && TYPE_MODE (vectype
) == TYPE_MODE_RAW (vectype
)
3359 && TYPE_ALIGN (vectype
) == 16
3360 && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype
)),
3361 BYTES_PER_SVE_VECTOR
));
3366 scalar_mode elmode
= SCALAR_TYPE_MODE (eltype
);
3367 unsigned int elbytes
= GET_MODE_SIZE (elmode
);
3368 poly_uint64 nunits
= exact_div (BYTES_PER_SVE_VECTOR
, elbytes
);
3370 = aarch64_sve_data_mode (elmode
, nunits
).require ();
3371 vectype
= build_vector_type_for_mode (eltype
, mode
);
3372 gcc_assert (VECTOR_MODE_P (TYPE_MODE (vectype
))
3373 && TYPE_MODE (vectype
) == mode
3374 && TYPE_MODE_RAW (vectype
) == mode
3375 && TYPE_ALIGN (vectype
) == 128
3376 && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype
)),
3377 BITS_PER_SVE_VECTOR
));
3380 vectype
= build_distinct_type_copy (vectype
);
3381 gcc_assert (vectype
== TYPE_MAIN_VARIANT (vectype
));
3382 SET_TYPE_STRUCTURAL_EQUALITY (vectype
);
3383 TYPE_ARTIFICIAL (vectype
) = 1;
3384 TYPE_INDIVISIBLE_P (vectype
) = 1;
3385 add_sve_type_attribute (vectype
, num_zr
, num_pr
,
3386 vector_types
[i
].mangled_name
,
3387 vector_types
[i
].acle_name
);
3388 make_type_sizeless (vectype
);
3389 abi_vector_types
[i
] = vectype
;
3390 lang_hooks
.types
.register_builtin_type (vectype
,
3391 vector_types
[i
].abi_name
);
3395 /* Initialize all compiler built-ins related to SVE that should be
3396 defined at start-up. */
3401 register_builtin_types ();
3403 handle_arm_sve_h ();
3406 /* Register vector type TYPE under its arm_sve.h name. */
3408 register_vector_type (vector_type_index type
)
3410 tree vectype
= abi_vector_types
[type
];
3411 tree id
= get_identifier (vector_types
[type
].acle_name
);
3412 tree decl
= build_decl (input_location
, TYPE_DECL
, id
, vectype
);
3413 decl
= lang_hooks
.decls
.pushdecl (decl
);
3415 /* Record the new ACLE type if pushdecl succeeded without error. Use
3416 the ABI type otherwise, so that the type we record at least has the
3417 right form, even if it doesn't have the right name. This should give
3418 better error recovery behavior than installing error_mark_node or
3419 installing an incorrect type. */
3421 && TREE_CODE (decl
) == TYPE_DECL
3422 && TREE_TYPE (decl
) != error_mark_node
3423 && TYPE_MAIN_VARIANT (TREE_TYPE (decl
)) == vectype
)
3424 vectype
= TREE_TYPE (decl
);
3425 acle_vector_types
[0][type
] = vectype
;
3428 /* Register the tuple type that contains NUM_VECTORS vectors of type TYPE. */
3430 register_tuple_type (unsigned int num_vectors
, vector_type_index type
)
3432 tree tuple_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3434 /* Work out the structure name. */
3435 char buffer
[sizeof ("svbfloat16x4_t")];
3436 const char *vector_type_name
= vector_types
[type
].acle_name
;
3437 snprintf (buffer
, sizeof (buffer
), "%.*sx%d_t",
3438 (int) strlen (vector_type_name
) - 2, vector_type_name
,
3441 /* The contents of the type are opaque, so we can define them in any
3442 way that maps to the correct ABI type.
3444 Here we choose to use the same layout as for arm_neon.h, but with
3445 "__val" instead of "val":
3447 struct svfooxN_t { svfoo_t __val[N]; };
3449 (It wouldn't be possible to write that directly in C or C++ for
3450 sizeless types, but that's not a problem for this function.)
3452 Using arrays simplifies the handling of svget and svset for variable
3454 tree vector_type
= acle_vector_types
[0][type
];
3455 tree array_type
= build_array_type_nelts (vector_type
, num_vectors
);
3456 gcc_assert (VECTOR_MODE_P (TYPE_MODE (array_type
))
3457 && TYPE_MODE_RAW (array_type
) == TYPE_MODE (array_type
)
3458 && TYPE_ALIGN (array_type
) == 128);
3460 tree field
= build_decl (input_location
, FIELD_DECL
,
3461 get_identifier ("__val"), array_type
);
3462 DECL_FIELD_CONTEXT (field
) = tuple_type
;
3463 TYPE_FIELDS (tuple_type
) = field
;
3464 add_sve_type_attribute (tuple_type
, num_vectors
, 0, NULL
, buffer
);
3465 make_type_sizeless (tuple_type
);
3466 layout_type (tuple_type
);
3467 gcc_assert (VECTOR_MODE_P (TYPE_MODE (tuple_type
))
3468 && TYPE_MODE_RAW (tuple_type
) == TYPE_MODE (tuple_type
)
3469 && TYPE_ALIGN (tuple_type
) == 128);
3471 tree decl
= build_decl (input_location
, TYPE_DECL
,
3472 get_identifier (buffer
), tuple_type
);
3473 TYPE_NAME (tuple_type
) = decl
;
3474 TYPE_STUB_DECL (tuple_type
) = decl
;
3475 lang_hooks
.decls
.pushdecl (decl
);
3476 /* ??? Undo the effect of set_underlying_type for C. The C frontend
3477 doesn't recognize DECL as a built-in because (as intended) the decl has
3478 a real location instead of BUILTINS_LOCATION. The frontend therefore
3479 treats the decl like a normal C "typedef struct foo foo;", expecting
3480 the type for tag "struct foo" to have a dummy unnamed TYPE_DECL instead
3481 of the named one we attached above. It then sets DECL_ORIGINAL_TYPE
3482 on the supposedly unnamed decl, creating a circularity that upsets
3485 We don't want to follow the normal C model and create "struct foo"
3486 tags for tuple types since (a) the types are supposed to be opaque
3487 and (b) they couldn't be defined as a real struct anyway. Treating
3488 the TYPE_DECLs as "typedef struct foo foo;" without creating
3489 "struct foo" would lead to confusing error messages. */
3490 DECL_ORIGINAL_TYPE (decl
) = NULL_TREE
;
3492 acle_vector_types
[num_vectors
- 1][type
] = tuple_type
;
3495 /* Register the svpattern enum. */
3497 register_svpattern ()
3499 auto_vec
<string_int_pair
, 32> values
;
3500 #define PUSH(UPPER, LOWER, VALUE) \
3501 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE));
3502 AARCH64_FOR_SVPATTERN (PUSH
)
3505 acle_svpattern
= lang_hooks
.types
.simulate_enum_decl (input_location
,
3506 "svpattern", &values
);
3509 /* Register the svprfop enum. */
3513 auto_vec
<string_int_pair
, 16> values
;
3514 #define PUSH(UPPER, LOWER, VALUE) \
3515 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE));
3516 AARCH64_FOR_SVPRFOP (PUSH
)
3519 acle_svprfop
= lang_hooks
.types
.simulate_enum_decl (input_location
,
3520 "svprfop", &values
);
3523 /* Implement #pragma GCC aarch64 "arm_sve.h". */
3529 error ("duplicate definition of %qs", "arm_sve.h");
3535 /* Define the vector and tuple types. */
3536 for (unsigned int type_i
= 0; type_i
< NUM_VECTOR_TYPES
; ++type_i
)
3538 vector_type_index type
= vector_type_index (type_i
);
3539 register_vector_type (type
);
3540 if (type
!= VECTOR_TYPE_svbool_t
)
3541 for (unsigned int count
= 2; count
<= MAX_TUPLE_SIZE
; ++count
)
3542 register_tuple_type (count
, type
);
3545 /* Define the enums. */
3546 register_svpattern ();
3547 register_svprfop ();
3549 /* Define the functions. */
3550 function_table
= new hash_table
<registered_function_hasher
> (1023);
3551 function_builder builder
;
3552 for (unsigned int i
= 0; i
< ARRAY_SIZE (function_groups
); ++i
)
3553 builder
.register_function_group (function_groups
[i
]);
3556 /* Return the function decl with SVE function subcode CODE, or error_mark_node
3557 if no such function exists. */
3559 builtin_decl (unsigned int code
, bool)
3561 if (code
>= vec_safe_length (registered_functions
))
3562 return error_mark_node
;
3563 return (*registered_functions
)[code
]->decl
;
3566 /* If we're implementing manual overloading, check whether the SVE
3567 function with subcode CODE is overloaded, and if so attempt to
3568 determine the corresponding non-overloaded function. The call
3569 occurs at location LOCATION and has the arguments given by ARGLIST.
3571 If the call is erroneous, report an appropriate error and return
3572 error_mark_node. Otherwise, if the function is overloaded, return
3573 the decl of the non-overloaded function. Return NULL_TREE otherwise,
3574 indicating that the call should be processed in the normal way. */
3576 resolve_overloaded_builtin (location_t location
, unsigned int code
,
3577 vec
<tree
, va_gc
> *arglist
)
3579 if (code
>= vec_safe_length (registered_functions
))
3582 registered_function
&rfn
= *(*registered_functions
)[code
];
3583 if (rfn
.overloaded_p
)
3584 return function_resolver (location
, rfn
.instance
, rfn
.decl
,
3585 *arglist
).resolve ();
3589 /* Perform any semantic checks needed for a call to the SVE function
3590 with subcode CODE, such as testing for integer constant expressions.
3591 The call occurs at location LOCATION and has NARGS arguments,
3592 given by ARGS. FNDECL is the original function decl, before
3593 overload resolution.
3595 Return true if the call is valid, otherwise report a suitable error. */
3597 check_builtin_call (location_t location
, vec
<location_t
>, unsigned int code
,
3598 tree fndecl
, unsigned int nargs
, tree
*args
)
3600 const registered_function
&rfn
= *(*registered_functions
)[code
];
3601 if (!check_required_extensions (location
, rfn
.decl
, rfn
.required_extensions
))
3603 return function_checker (location
, rfn
.instance
, fndecl
,
3604 TREE_TYPE (rfn
.decl
), nargs
, args
).check ();
3607 /* Attempt to fold STMT, given that it's a call to the SVE function
3608 with subcode CODE. Return the new statement on success and null
3609 on failure. Insert any other new statements at GSI. */
3611 gimple_fold_builtin (unsigned int code
, gimple_stmt_iterator
*gsi
, gcall
*stmt
)
3613 registered_function
&rfn
= *(*registered_functions
)[code
];
3614 return gimple_folder (rfn
.instance
, rfn
.decl
, gsi
, stmt
).fold ();
3617 /* Expand a call to the SVE function with subcode CODE. EXP is the call
3618 expression and TARGET is the preferred location for the result.
3619 Return the value of the lhs. */
3621 expand_builtin (unsigned int code
, tree exp
, rtx target
)
3623 registered_function
&rfn
= *(*registered_functions
)[code
];
3624 if (!check_required_extensions (EXPR_LOCATION (exp
), rfn
.decl
,
3625 rfn
.required_extensions
))
3627 return function_expander (rfn
.instance
, rfn
.decl
, exp
, target
).expand ();
3630 /* If TYPE is a built-in type defined by the SVE ABI, return the mangled name,
3631 otherwise return NULL. */
3633 mangle_builtin_type (const_tree type
)
3635 /* ??? The C++ frontend normally strips qualifiers and attributes before
3636 calling this hook, adding separate mangling for attributes that affect
3637 type identity. Fortunately the type copy will have the same TYPE_NAME
3638 as the original, so we can get the attributes from there. */
3639 if (TYPE_NAME (type
) && TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
)
3640 type
= TREE_TYPE (TYPE_NAME (type
));
3641 if (tree attr
= lookup_sve_type_attribute (type
))
3642 if (tree id
= TREE_VALUE (chain_index (2, TREE_VALUE (attr
))))
3643 return IDENTIFIER_POINTER (id
);
3647 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. */
3649 builtin_type_p (const_tree type
)
3651 return lookup_sve_type_attribute (type
);
3654 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE.
3655 If so, store the number of constituent SVE vectors in *NUM_ZR and the
3656 number of constituent SVE predicates in *NUM_PR. */
3658 builtin_type_p (const_tree type
, unsigned int *num_zr
, unsigned int *num_pr
)
3660 if (tree attr
= lookup_sve_type_attribute (type
))
3662 tree num_zr_node
= TREE_VALUE (attr
);
3663 tree num_pr_node
= TREE_CHAIN (num_zr_node
);
3664 *num_zr
= tree_to_uhwi (TREE_VALUE (num_zr_node
));
3665 *num_pr
= tree_to_uhwi (TREE_VALUE (num_pr_node
));
3671 /* ATTRS is the attribute list for a sizeless SVE type. Return the
3672 attributes of the associated fixed-length SVE type, taking the
3673 "SVE type" attributes from NEW_SVE_TYPE_ARGS. */
3675 get_arm_sve_vector_bits_attributes (tree old_attrs
, tree new_sve_type_args
)
3677 tree new_attrs
= NULL_TREE
;
3678 tree
*ptr
= &new_attrs
;
3679 for (tree attr
= old_attrs
; attr
; attr
= TREE_CHAIN (attr
))
3681 tree name
= get_attribute_name (attr
);
3682 if (is_attribute_p ("SVE sizeless type", name
))
3685 tree args
= TREE_VALUE (attr
);
3686 if (is_attribute_p ("SVE type", name
))
3687 args
= new_sve_type_args
;
3688 *ptr
= tree_cons (TREE_PURPOSE (attr
), args
, NULL_TREE
);
3689 ptr
= &TREE_CHAIN (*ptr
);
3694 /* An attribute callback for the "arm_sve_vector_bits" attribute. */
3696 handle_arm_sve_vector_bits_attribute (tree
*node
, tree
, tree args
, int,
3699 *no_add_attrs
= true;
3702 tree attr
= lookup_sve_type_attribute (type
);
3705 error ("%qs applied to non-SVE type %qT", "arm_sve_vector_bits", type
);
3709 if (!VECTOR_TYPE_P (type
))
3711 error ("%qs applied to non-vector type %qT",
3712 "arm_sve_vector_bits", type
);
3716 if (!sizeless_type_p (type
))
3718 error ("%qs applied to type %qT, which already has a size",
3719 "arm_sve_vector_bits", type
);
3723 tree size
= TREE_VALUE (args
);
3724 if (TREE_CODE (size
) != INTEGER_CST
)
3726 error ("%qs requires an integer constant expression",
3727 "arm_sve_vector_bits");
3731 unsigned HOST_WIDE_INT value
= tree_to_uhwi (size
);
3732 if (maybe_ne (value
, BITS_PER_SVE_VECTOR
))
3734 warning (OPT_Wattributes
, "unsupported SVE vector size");
3738 /* Construct a new list of "SVE type" attribute arguments. */
3739 tree new_sve_type_args
= copy_list (TREE_VALUE (attr
));
3741 /* Mangle the type as an instance of the imaginary template:
3743 __SVE_VLS<typename, unsigned>
3745 where the first parameter is the SVE type and where the second
3746 parameter is the SVE vector length in bits. */
3747 tree mangled_name_node
= chain_index (2, new_sve_type_args
);
3748 const char *old_mangled_name
3749 = IDENTIFIER_POINTER (TREE_VALUE (mangled_name_node
));
3750 char *new_mangled_name
3751 = xasprintf ("9__SVE_VLSI%sLj%dEE", old_mangled_name
, (int) value
);
3752 TREE_VALUE (mangled_name_node
) = get_identifier (new_mangled_name
);
3753 free (new_mangled_name
);
3755 /* FIXME: The type ought to be a distinct copy in all cases, but
3756 currently that makes the C frontend reject conversions between
3757 svbool_t and its fixed-length variants. Using a type variant
3758 avoids that but means that we treat some ambiguous combinations
3761 tree base_type
= TYPE_MAIN_VARIANT (type
);
3762 if (lang_GNU_C () && VECTOR_BOOLEAN_TYPE_P (type
))
3763 new_type
= build_variant_type_copy (base_type
);
3765 new_type
= build_distinct_type_copy (base_type
);
3767 /* Construct a TYPE_DECL for the new type. This serves two purposes:
3769 - It ensures we don't print the original TYPE_DECL in error messages.
3770 Printing the original name would be confusing because there are
3771 situations in which the distinction between the original type and
3772 the new type matters. For example:
3774 __SVInt8_t __attribute__((arm_sve_vector_bits(512))) *a;
3779 is invalid in C++, but without this, we'd print both types in
3782 - Having a separate TYPE_DECL is necessary to ensure that C++
3783 mangling works correctly. See mangle_builtin_type for details.
3785 The name of the decl is something like:
3787 svint8_t __attribute__((arm_sve_vector_bits(512)))
3789 This is a compromise. It would be more accurate to use something like:
3791 __SVInt8_t __attribute__((arm_sve_vector_bits(512)))
3793 but the <arm_sve.h> name is likely to be more meaningful. */
3794 tree acle_name_node
= TREE_CHAIN (mangled_name_node
);
3795 const char *old_type_name
= IDENTIFIER_POINTER (TREE_VALUE (acle_name_node
));
3797 = xasprintf ("%s __attribute__((arm_sve_vector_bits(%d)))",
3798 old_type_name
, (int) value
);
3799 tree decl
= build_decl (BUILTINS_LOCATION
, TYPE_DECL
,
3800 get_identifier (new_type_name
), new_type
);
3801 DECL_ARTIFICIAL (decl
) = 1;
3802 TYPE_NAME (new_type
) = decl
;
3803 free (new_type_name
);
3805 /* Allow the GNU vector extensions to be applied to vectors.
3806 The extensions aren't yet defined for packed predicates,
3807 so continue to treat them as abstract entities for now. */
3808 if (!VECTOR_BOOLEAN_TYPE_P (new_type
))
3809 TYPE_INDIVISIBLE_P (new_type
) = 0;
3811 /* The new type is a normal sized type; it doesn't have the same
3812 restrictions as sizeless types. */
3813 TYPE_ATTRIBUTES (new_type
)
3814 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (new_type
),
3817 /* Apply the relevant attributes, qualifiers and alignment of TYPE,
3818 if they differ from the original (sizeless) BASE_TYPE. */
3819 if (TYPE_ATTRIBUTES (base_type
) != TYPE_ATTRIBUTES (type
)
3820 || TYPE_QUALS (base_type
) != TYPE_QUALS (type
))
3823 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (type
),
3825 new_type
= build_type_attribute_qual_variant (new_type
, attrs
,
3828 if (TYPE_ALIGN (base_type
) != TYPE_ALIGN (type
))
3829 new_type
= build_aligned_type (new_type
, TYPE_ALIGN (type
));
3835 /* Implement TARGET_VERIFY_TYPE_CONTEXT for SVE types. */
3837 verify_type_context (location_t loc
, type_context_kind context
,
3838 const_tree type
, bool silent_p
)
3840 if (!sizeless_type_p (type
))
3846 case TCTX_STATIC_STORAGE
:
3848 error_at (loc
, "SVE type %qT does not have a fixed size", type
);
3853 error_at (loc
, "SVE type %qT does not have a defined alignment", type
);
3856 case TCTX_THREAD_STORAGE
:
3858 error_at (loc
, "variables of type %qT cannot have thread-local"
3859 " storage duration", type
);
3862 case TCTX_POINTER_ARITH
:
3864 error_at (loc
, "arithmetic on pointer to SVE type %qT", type
);
3870 else if (lang_GNU_CXX ())
3871 error_at (loc
, "member variables cannot have SVE type %qT", type
);
3873 error_at (loc
, "fields cannot have SVE type %qT", type
);
3876 case TCTX_ARRAY_ELEMENT
:
3878 error_at (loc
, "array elements cannot have SVE type %qT", type
);
3881 case TCTX_ALLOCATION
:
3883 error_at (loc
, "cannot allocate objects with SVE type %qT", type
);
3886 case TCTX_DEALLOCATION
:
3888 error_at (loc
, "cannot delete objects with SVE type %qT", type
);
3891 case TCTX_EXCEPTIONS
:
3893 error_at (loc
, "cannot throw or catch SVE type %qT", type
);
3896 case TCTX_CAPTURE_BY_COPY
:
3898 error_at (loc
, "capture by copy of SVE type %qT", type
);
3906 using namespace aarch64_sve
;
3909 gt_ggc_mx (function_instance
*)
3914 gt_pch_nx (function_instance
*)
3919 gt_pch_nx (function_instance
*, gt_pointer_operator
, void *)
3923 #include "gt-aarch64-sve-builtins.h"