VECTOR_MODE (FLOAT, DF, 1); /* V1DF. */
VECTOR_MODE (FLOAT, HF, 2); /* V2HF. */
+
+/* Integer vector modes used to represent intermediate widened values in some
+ instructions. Not intended to be moved to and from registers or memory. */
+VECTOR_MODE (INT, HI, 16); /* V16HI. */
+VECTOR_MODE (INT, SI, 8); /* V8SI. */
+VECTOR_MODE (INT, DI, 4); /* V4DI. */
+VECTOR_MODE (INT, TI, 2); /* V2TI. */
+
/* Oct Int: 256-bit integer mode needed for 32-byte vector arguments. */
INT_MODE (OI, 32);
bool aarch64_const_vec_all_same_int_p (rtx, HOST_WIDE_INT);
bool aarch64_const_vec_all_same_in_range_p (rtx, HOST_WIDE_INT,
HOST_WIDE_INT);
+bool aarch64_const_vec_rnd_cst_p (rtx, rtx);
+bool aarch64_const_vec_rsra_rnd_imm_p (rtx);
bool aarch64_constant_address_p (rtx);
bool aarch64_emit_approx_div (rtx, rtx, rtx);
bool aarch64_emit_approx_sqrt (rtx, rtx, bool);
[(set_attr "type" "neon_compare<q>,neon_shift_imm<q>")]
)
-(define_insn "*aarch64_simd_sra<mode>"
+(define_insn "aarch64_<sra_op>sra_n<mode>_insn"
[(set (match_operand:VDQ_I 0 "register_operand" "=w")
(plus:VDQ_I
(SHIFTRT:VDQ_I
- (match_operand:VDQ_I 1 "register_operand" "w")
- (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr"))
- (match_operand:VDQ_I 3 "register_operand" "0")))]
+ (match_operand:VDQ_I 2 "register_operand" "w")
+ (match_operand:VDQ_I 3 "aarch64_simd_rshift_imm"))
+ (match_operand:VDQ_I 1 "register_operand" "0")))]
"TARGET_SIMD"
- "<sra_op>sra\t%0.<Vtype>, %1.<Vtype>, %2"
+ "<sra_op>sra\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
[(set_attr "type" "neon_shift_acc<q>")]
)
+(define_insn "aarch64_<sra_op>rsra_n<mode>_insn"
+ [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
+ (plus:VSDQ_I_DI
+ (truncate:VSDQ_I_DI
+ (SHIFTRT:<V2XWIDE>
+ (plus:<V2XWIDE>
+ (<SHIFTEXTEND>:<V2XWIDE>
+ (match_operand:VSDQ_I_DI 2 "register_operand" "w"))
+ (match_operand:<V2XWIDE> 4 "aarch64_simd_rsra_rnd_imm_vec"))
+ (match_operand:VSDQ_I_DI 3 "aarch64_simd_shift_imm_<vec_or_offset>_<Vel>")))
+ (match_operand:VSDQ_I_DI 1 "register_operand" "0")))]
+ "TARGET_SIMD
+ && aarch64_const_vec_rnd_cst_p (operands[4], operands[3])"
+ "<sra_op>rsra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
+ [(set_attr "type" "neon_shift_acc<q>")]
+)
+
+(define_expand "aarch64_<sra_op>sra_n<mode>"
+ [(set (match_operand:VDQ_I 0 "register_operand")
+ (plus:VDQ_I
+ (SHIFTRT:VDQ_I
+ (match_operand:VDQ_I 2 "register_operand")
+ (match_operand:SI 3 "aarch64_simd_shift_imm_offset_<ve_mode>"))
+ (match_operand:VDQ_I 1 "register_operand")))]
+ "TARGET_SIMD"
+ {
+ operands[3]
+ = aarch64_simd_gen_const_vector_dup (<MODE>mode, UINTVAL (operands[3]));
+ }
+)
+
+(define_expand "aarch64_<sra_op>rsra_n<mode>"
+ [(match_operand:VSDQ_I_DI 0 "register_operand")
+ (match_operand:VSDQ_I_DI 1 "register_operand")
+ (SHIFTRT:VSDQ_I_DI
+ (match_operand:VSDQ_I_DI 2 "register_operand")
+ (match_operand:SI 3 "aarch64_simd_shift_imm_offset_<ve_mode>"))]
+ "TARGET_SIMD"
+ {
+ /* Use this expander to create the rounding constant vector, which is
+ 1 << (shift - 1). Use wide_int here to ensure that the right TImode
+ RTL is generated when handling the DImode expanders. */
+ int prec = GET_MODE_UNIT_PRECISION (<V2XWIDE>mode);
+ wide_int rnd_wi = wi::set_bit_in_zero (INTVAL (operands[3]) - 1, prec);
+ rtx shft = gen_int_mode (INTVAL (operands[3]), DImode);
+ rtx rnd = immed_wide_int_const (rnd_wi, GET_MODE_INNER (<V2XWIDE>mode));
+ if (VECTOR_MODE_P (<MODE>mode))
+ {
+ shft = gen_const_vec_duplicate (<MODE>mode, shft);
+ rnd = gen_const_vec_duplicate (<V2XWIDE>mode, rnd);
+ }
+
+ emit_insn (gen_aarch64_<sra_op>rsra_n<mode>_insn (operands[0], operands[1],
+ operands[2], shft, rnd));
+ DONE;
+ }
+)
+
(define_insn "aarch64_simd_imm_shl<mode><vczle><vczbe>"
[(set (match_operand:VDQ_I 0 "register_operand" "=w")
(ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
;; v(r)sra_n
-(define_insn "aarch64_<sur>sra_n<mode>"
- [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
- (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
- (match_operand:VSDQ_I_DI 2 "register_operand" "w")
+(define_insn "aarch64_<sur>sra_ndi"
+ [(set (match_operand:DI 0 "register_operand" "=w")
+ (unspec:DI [(match_operand:DI 1 "register_operand" "0")
+ (match_operand:DI 2 "register_operand" "w")
(match_operand:SI 3
- "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
+ "aarch64_simd_shift_imm_offset_di" "i")]
VSRA))]
"TARGET_SIMD"
- "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
- [(set_attr "type" "neon_shift_acc<q>")]
+ "<sur>sra\\t%d0, %d2, %3"
+ [(set_attr "type" "neon_shift_acc")]
)
;; vs<lr>i_n
const unsigned int VEC_ANY_DATA = VEC_ADVSIMD | VEC_SVE_DATA;
/* Return a set of flags describing the vector properties of mode MODE.
- Ignore modes that are not supported by the current target. */
+ If ANY_TARGET_P is false (the default), ignore modes that are not supported
+ by the current target. Otherwise categorize the modes that can be used
+ with the set of all targets supported by the port. */
+
static unsigned int
-aarch64_classify_vector_mode (machine_mode mode)
+aarch64_classify_vector_mode (machine_mode mode, bool any_target_p = false)
{
if (aarch64_sve_pred_mode_p (mode))
return VEC_SVE_PRED;
case E_VNx4BFmode:
/* Partial SVE SF vector. */
case E_VNx2SFmode:
- return TARGET_SVE ? VEC_SVE_DATA | VEC_PARTIAL : 0;
+ return (TARGET_SVE || any_target_p) ? VEC_SVE_DATA | VEC_PARTIAL : 0;
case E_VNx16QImode:
case E_VNx8HImode:
case E_VNx8HFmode:
case E_VNx4SFmode:
case E_VNx2DFmode:
- return TARGET_SVE ? VEC_SVE_DATA : 0;
+ return (TARGET_SVE || any_target_p) ? VEC_SVE_DATA : 0;
/* x2 SVE vectors. */
case E_VNx32QImode:
case E_VNx32HFmode:
case E_VNx16SFmode:
case E_VNx8DFmode:
- return TARGET_SVE ? VEC_SVE_DATA | VEC_STRUCT : 0;
+ return (TARGET_SVE || any_target_p) ? VEC_SVE_DATA | VEC_STRUCT : 0;
case E_OImode:
case E_CImode:
case E_XImode:
- return TARGET_FLOAT ? VEC_ADVSIMD | VEC_STRUCT : 0;
+ return (TARGET_FLOAT || any_target_p) ? VEC_ADVSIMD | VEC_STRUCT : 0;
/* Structures of 64-bit Advanced SIMD vectors. */
case E_V2x8QImode:
case E_V4x4HFmode:
case E_V4x2SFmode:
case E_V4x1DFmode:
- return TARGET_FLOAT ? VEC_ADVSIMD | VEC_STRUCT | VEC_PARTIAL : 0;
+ return (TARGET_FLOAT || any_target_p)
+ ? VEC_ADVSIMD | VEC_STRUCT | VEC_PARTIAL : 0;
/* Structures of 128-bit Advanced SIMD vectors. */
case E_V2x16QImode:
case E_V4x8HFmode:
case E_V4x4SFmode:
case E_V4x2DFmode:
- return TARGET_FLOAT ? VEC_ADVSIMD | VEC_STRUCT : 0;
+ return (TARGET_FLOAT || any_target_p) ? VEC_ADVSIMD | VEC_STRUCT : 0;
/* 64-bit Advanced SIMD vectors. */
case E_V8QImode:
case E_V8BFmode:
case E_V4SFmode:
case E_V2DFmode:
- return TARGET_FLOAT ? VEC_ADVSIMD : 0;
+ return (TARGET_FLOAT || any_target_p) ? VEC_ADVSIMD : 0;
default:
return 0;
return -1;
}
+/* Return true if X is a CONST_INT, CONST_WIDE_INT or a constant vector
+ duplicate of such constants. If so, store in RET_WI the wide_int
+ representation of the constant paired with the inner mode of the vector mode
+ or TImode for scalar X constants. */
+
+static bool
+aarch64_extract_vec_duplicate_wide_int (rtx x, wide_int *ret_wi)
+{
+ rtx elt = unwrap_const_vec_duplicate (x);
+ if (!CONST_SCALAR_INT_P (elt))
+ return false;
+ scalar_mode smode
+ = CONST_SCALAR_INT_P (x) ? TImode : GET_MODE_INNER (GET_MODE (x));
+ *ret_wi = rtx_mode_t (elt, smode);
+ return true;
+}
+
+/* Return true if X is a TImode constant or a constant vector of integer
+ immediates that represent the rounding constant used in the RSRA
+ instructions.
+ The accepted form of the constant is (1 << (C - 1)) where C is within
+ [1, MODE_WIDTH/2]. */
+
+bool
+aarch64_const_vec_rsra_rnd_imm_p (rtx x)
+{
+ wide_int rnd_cst;
+ if (!aarch64_extract_vec_duplicate_wide_int (x, &rnd_cst))
+ return false;
+ int log2 = wi::exact_log2 (rnd_cst);
+ if (log2 < 0)
+ return false;
+ return IN_RANGE (log2, 0, rnd_cst.get_precision () / 2 - 1);
+}
+
+/* Return true if RND is a constant vector of integer rounding constants
+ corresponding to a constant vector of shifts, SHIFT.
+ The relationship should be RND == (1 << (SHIFT - 1)). */
+
+bool
+aarch64_const_vec_rnd_cst_p (rtx rnd, rtx shift)
+{
+ wide_int rnd_cst, shft_cst;
+ if (!aarch64_extract_vec_duplicate_wide_int (rnd, &rnd_cst)
+ || !aarch64_extract_vec_duplicate_wide_int (shift, &shft_cst))
+ return false;
+
+ return rnd_cst == (wi::shwi (1, rnd_cst.get_precision ()) << (shft_cst - 1));
+}
+
bool
aarch64_const_vec_all_same_in_range_p (rtx x,
HOST_WIDE_INT minval,
return vec_flags != 0 && (vec_flags & VEC_STRUCT) == 0;
}
+/* Implements target hook vector_mode_supported_any_target_p. */
+static bool
+aarch64_vector_mode_supported_any_target_p (machine_mode mode)
+{
+ unsigned int vec_flags = aarch64_classify_vector_mode (mode, true);
+ return vec_flags != 0 && (vec_flags & VEC_STRUCT) == 0;
+}
+
/* Return the full-width SVE vector mode for element mode MODE, if one
exists. */
opt_machine_mode
#undef TARGET_VECTOR_MODE_SUPPORTED_P
#define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
+#undef TARGET_VECTOR_MODE_SUPPORTED_ANY_TARGET_P
+#define TARGET_VECTOR_MODE_SUPPORTED_ANY_TARGET_P aarch64_vector_mode_supported_any_target_p
+
#undef TARGET_COMPATIBLE_VECTOR_TYPES_P
#define TARGET_COMPATIBLE_VECTOR_TYPES_P aarch64_compatible_vector_types_p
UNSPEC_SQXTUN ; Used in aarch64-simd.md.
UNSPEC_SSRA ; Used in aarch64-simd.md.
UNSPEC_USRA ; Used in aarch64-simd.md.
- UNSPEC_SRSRA ; Used in aarch64-simd.md.
- UNSPEC_URSRA ; Used in aarch64-simd.md.
UNSPEC_SRSHR ; Used in aarch64-simd.md.
UNSPEC_URSHR ; Used in aarch64-simd.md.
UNSPEC_SQSHLU ; Used in aarch64-simd.md.
(VNx16BI "VNx8BI") (VNx8BI "VNx4BI")
(VNx4BI "VNx2BI")])
+;; Modes with the same number of elements but strictly 2x the width.
+(define_mode_attr V2XWIDE [(V8QI "V8HI") (V4HI "V4SI")
+ (V16QI "V16HI") (V8HI "V8SI")
+ (V2SI "V2DI") (V4SI "V4DI")
+ (V2DI "V2TI") (DI "TI")])
+
;; Predicate mode associated with VWIDE.
(define_mode_attr VWIDE_PRED [(VNx8HF "VNx4BI") (VNx4SF "VNx2BI")])
;; The constraint to use for an SVE FCMLA lane index.
(define_mode_attr sve_lane_pair_con [(VNx8HF "y") (VNx4SF "x")])
+(define_mode_attr vec_or_offset [(V8QI "vec") (V16QI "vec") (V4HI "vec")
+ (V8HI "vec") (V2SI "vec") (V4SI "vec")
+ (V2DI "vec") (DI "offset")])
+
;; -------------------------------------------------------------------
;; Code Iterators
;; -------------------------------------------------------------------
(ss_minus "sub")
(us_minus "sub")])
+(define_code_attr SHIFTEXTEND [(ashiftrt "sign_extend") (lshiftrt "zero_extend")])
+
;; For comparison operators we use the FCM* and CM* instructions.
;; As there are no CMLE or CMLT instructions which act on 3 vector
;; operands, we must use CMGE or CMGT and swap the order of the
(define_int_iterator VQSHL [UNSPEC_SQSHL UNSPEC_UQSHL
UNSPEC_SQRSHL UNSPEC_UQRSHL])
-(define_int_iterator VSRA [UNSPEC_SSRA UNSPEC_USRA
- UNSPEC_SRSRA UNSPEC_URSRA])
+(define_int_iterator VSRA [UNSPEC_SSRA UNSPEC_USRA])
(define_int_iterator VSLRI [UNSPEC_SSLI UNSPEC_USLI
UNSPEC_SSRI UNSPEC_USRI])
(UNSPEC_SSLI "s") (UNSPEC_USLI "u")
(UNSPEC_SSRI "s") (UNSPEC_USRI "u")
(UNSPEC_USRA "u") (UNSPEC_SSRA "s")
- (UNSPEC_URSRA "ur") (UNSPEC_SRSRA "sr")
(UNSPEC_URSHR "ur") (UNSPEC_SRSHR "sr")
(UNSPEC_SQSHLU "s") (UNSPEC_SQSHL "s")
(UNSPEC_UQSHL "u")
(and (match_code "const_vector")
(match_test "aarch64_const_vec_all_same_in_range_p (op, 1, 64)")))
+(define_predicate "aarch64_simd_rsra_rnd_imm_vec"
+ (and (match_code "const_vector,const_int,const_wide_int")
+ (match_test "aarch64_const_vec_rsra_rnd_imm_p (op)")))
+
(define_predicate "aarch64_simd_rshrn_imm_vec"
(and (match_code "const_vector")
(match_test "aarch64_const_vec_all_same_in_range_p (op, 1,
@end deftypefn
@deftypefn {Target Hook} bool TARGET_VECTOR_MODE_SUPPORTED_P (machine_mode @var{mode})
-Define this to return nonzero if the port is prepared to handle
+Define this to return nonzero if the current target is prepared to handle
insns involving vector mode @var{mode}. At the very least, it
must have move patterns for this mode.
@end deftypefn
+@deftypefn {Target Hook} bool TARGET_VECTOR_MODE_SUPPORTED_ANY_TARGET_P (machine_mode @var{mode})
+Define this to return nonzero if the port is prepared to handle
+insns involving vector mode @var{mode} in any target configuration.
+Returning @var{true} means that the mode can be used as the @samp{TYPE_MODE}
+for vector types.
+
+The default version of this hook returns true. The final mode assigned to
+@samp{TYPE_MODE} will also be checked against
+@code{TARGET_VECTOR_MODE_SUPPORTED_P} to take target configuration into
+account.
+@end deftypefn
+
@deftypefn {Target Hook} bool TARGET_COMPATIBLE_VECTOR_TYPES_P (const_tree @var{type1}, const_tree @var{type2})
Return true if there is no target-specific reason for treating
vector types @var{type1} and @var{type2} as distinct types. The caller
@hook TARGET_VECTOR_MODE_SUPPORTED_P
+@hook TARGET_VECTOR_MODE_SUPPORTED_ANY_TARGET_P
+
@hook TARGET_COMPATIBLE_VECTOR_TYPES_P
@hook TARGET_ARRAY_MODE
else
mode = MIN_MODE_VECTOR_INT;
- /* Do not check vector_mode_supported_p here. We'll do that
- later in vector_type_mode. */
+ /* Only check the broader vector_mode_supported_any_target_p here.
+ We'll filter through target-specific availability and
+ vector_mode_supported_p later in vector_type_mode. */
FOR_EACH_MODE_FROM (mode, mode)
if (known_eq (GET_MODE_NUNITS (mode), nunits)
- && GET_MODE_INNER (mode) == innermode)
+ && GET_MODE_INNER (mode) == innermode
+ && targetm.vector_mode_supported_any_target_p (mode))
return mode;
/* For integers, try mapping it to a same-sized scalar mode. */
for further details. */
DEFHOOK
(vector_mode_supported_p,
- "Define this to return nonzero if the port is prepared to handle\n\
+ "Define this to return nonzero if the current target is prepared to handle\n\
insns involving vector mode @var{mode}. At the very least, it\n\
must have move patterns for this mode.",
bool, (machine_mode mode),
hook_bool_mode_false)
+DEFHOOK
+(vector_mode_supported_any_target_p,
+ "Define this to return nonzero if the port is prepared to handle\n\
+insns involving vector mode @var{mode} in any target configuration.\n\
+Returning @var{true} means that the mode can be used as the @samp{TYPE_MODE}\n\
+for vector types.\n\
+\n\
+The default version of this hook returns true. The final mode assigned to\n\
+@samp{TYPE_MODE} will also be checked against\n\
+@code{TARGET_VECTOR_MODE_SUPPORTED_P} to take target configuration into\n\
+account.",
+ bool, (machine_mode mode),
+ hook_bool_mode_true)
+
DEFHOOK
(compatible_vector_types_p,
"Return true if there is no target-specific reason for treating\n\