AARCH64_WSRF,
AARCH64_WSRF64,
AARCH64_WSR128,
+ AARCH64_PLD,
+ AARCH64_PLDX,
+ AARCH64_PLI,
+ AARCH64_PLIX,
AARCH64_BUILTIN_MAX
};
= build_function_type_list (void_type_node, const_char_ptr_type,
uint128_type_node, NULL);
AARCH64_INIT_RWSR_BUILTINS_DECL (WSR128, wsr128, fntype);
+}
+
+/* Add builtins for data and instrution prefetch. */
+static void
+aarch64_init_prefetch_builtin (void)
+{
+#define AARCH64_INIT_PREFETCH_BUILTIN(INDEX, N) \
+ aarch64_builtin_decls[INDEX] = \
+ aarch64_general_add_builtin ("__builtin_aarch64_" N, ftype, INDEX)
+
+ tree ftype;
+ tree cv_argtype;
+ cv_argtype = build_qualified_type (void_type_node, TYPE_QUAL_CONST
+ | TYPE_QUAL_VOLATILE);
+ cv_argtype = build_pointer_type (cv_argtype);
+
+ ftype = build_function_type_list (void_type_node, cv_argtype, NULL);
+ AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLD, "pld");
+ AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLI, "pli");
+
+ ftype = build_function_type_list (void_type_node, unsigned_type_node,
+ unsigned_type_node, unsigned_type_node,
+ cv_argtype, NULL);
+ AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLDX, "pldx");
+ ftype = build_function_type_list (void_type_node, unsigned_type_node,
+ unsigned_type_node, cv_argtype, NULL);
+ AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLIX, "plix");
}
/* Initialize the memory tagging extension (MTE) builtins. */
aarch64_init_data_intrinsics ();
aarch64_init_rwsr_builtins ();
+ aarch64_init_prefetch_builtin ();
tree ftype_jcvt
= build_function_type_list (intSI_type_node, double_type_node, NULL);
}
}
+/* Ensure argument ARGNO in EXP represents a const-type argument in the range
+ [MINVAL, MAXVAL). */
+static HOST_WIDE_INT
+require_const_argument (tree exp, unsigned int argno, HOST_WIDE_INT minval,
+ HOST_WIDE_INT maxval)
+{
+ maxval--;
+ tree arg = CALL_EXPR_ARG (exp, argno);
+ if (TREE_CODE (arg) != INTEGER_CST)
+ error_at (EXPR_LOCATION (exp), "Constant-type argument expected");
+
+ auto argval = wi::to_widest (arg);
+
+ if (argval < minval || argval > maxval)
+ error_at (EXPR_LOCATION (exp),
+ "argument %d must be a constant immediate "
+ "in range [%wd,%wd]", argno + 1, minval, maxval);
+
+ HOST_WIDE_INT retval = argval.to_shwi ();
+ return retval;
+}
+
+
+/* Expand a prefetch builtin EXP. */
+void
+aarch64_expand_prefetch_builtin (tree exp, int fcode)
+{
+ int kind_id = -1;
+ int level_id = -1;
+ int rettn_id = -1;
+ char prfop[11];
+ class expand_operand ops[2];
+
+ static const char *kind_s[] = {"PLD", "PST", "PLI"};
+ static const char *level_s[] = {"L1", "L2", "L3", "SLC"};
+ static const char *rettn_s[] = {"KEEP", "STRM"};
+
+ /* Each of the four prefetch builtins takes a different number of arguments,
+ but proceeds to call the PRFM insn which requires 4 pieces of information
+ to be fully defined. Where one of these takes less than 4 arguments, set
+ sensible defaults. */
+ switch (fcode)
+ {
+ case AARCH64_PLDX:
+ break;
+ case AARCH64_PLIX:
+ kind_id = 2;
+ break;
+ case AARCH64_PLI:
+ case AARCH64_PLD:
+ kind_id = (fcode == AARCH64_PLD) ? 0 : 2;
+ level_id = 0;
+ rettn_id = 0;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ /* Any -1 id variable is to be user-supplied. Here we fill these in and run
+ bounds checks on them. "PLI" is used only implicitly by AARCH64_PLI &
+ AARCH64_PLIX, never explicitly. */
+ int argno = 0;
+ if (kind_id < 0)
+ kind_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (kind_s) - 1);
+ if (level_id < 0)
+ level_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (level_s));
+ if (rettn_id < 0)
+ rettn_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (rettn_s));
+ rtx address = expand_expr (CALL_EXPR_ARG (exp, argno), NULL_RTX, Pmode,
+ EXPAND_NORMAL);
+
+ if (seen_error ())
+ return;
+
+ sprintf (prfop, "%s%s%s", kind_s[kind_id],
+ level_s[level_id],
+ rettn_s[rettn_id]);
+
+ rtx const_str = rtx_alloc (CONST_STRING);
+ PUT_CODE (const_str, CONST_STRING);
+ XSTR (const_str, 0) = ggc_strdup (prfop);
+
+ create_fixed_operand (&ops[0], const_str);
+ create_address_operand (&ops[1], address);
+ maybe_expand_insn (CODE_FOR_aarch64_pldx, 2, ops);
+}
+
/* Expand an expression EXP that calls a MEMTAG built-in FCODE
with result going to TARGET. */
static rtx
case AARCH64_WSRF64:
case AARCH64_WSR128:
return aarch64_expand_rwsr_builtin (exp, target, fcode);
+ case AARCH64_PLD:
+ case AARCH64_PLDX:
+ case AARCH64_PLI:
+ case AARCH64_PLIX:
+ aarch64_expand_prefetch_builtin (exp, fcode);
+ return target;
}
if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX)
UNSPEC_SYSREG_RTI
UNSPEC_SYSREG_WDI
UNSPEC_SYSREG_WTI
+ UNSPEC_PLDX
;; Represents an SVE-style lane index, in which the indexing applies
;; within the containing 128-bit block.
UNSPEC_SVE_LANE_SELECT
[(set_attr "type" "load_4")]
)
+(define_insn "aarch64_pldx"
+ [(unspec [(match_operand 0 "" "")
+ (match_operand:DI 1 "aarch64_prefetch_operand" "Dp")] UNSPEC_PLDX)]
+ ""
+ {
+ operands[1] = gen_rtx_MEM (DImode, operands[1]);
+ return "prfm\\t%0, %1";
+ }
+ [(set_attr "type" "load_4")]
+)
+
(define_insn "trap"
[(trap_if (const_int 1) (const_int 8))]
""
#undef _GCC_ARM_ACLE_DATA_FN
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__pld (void const volatile *__addr)
+{
+ return __builtin_aarch64_pld (__addr);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__pli (void const volatile *__addr)
+{
+ return __builtin_aarch64_pli (__addr);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__plix (unsigned int __cache, unsigned int __rettn,
+ void const volatile *__addr)
+{
+ return __builtin_aarch64_plix (__cache, __rettn, __addr);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__pldx (unsigned int __access, unsigned int __cache, unsigned int __rettn,
+ void const volatile *__addr)
+{
+ return __builtin_aarch64_pldx (__access, __cache, __rettn, __addr);
+}
+
__extension__ extern __inline unsigned long
__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
__revl (unsigned long __value)
--- /dev/null
+/* { dg-do compile } */
+/* { dg-options "-march=armv8-a -O2" } */
+
+#include <arm_acle.h>
+
+/* Check that we can generate the immediate-offset addressing
+ mode for PRFM. */
+
+/* Access kind specifiers. */
+#define PLD 0
+#define PST 1
+/* Cache levels. */
+#define L1 0
+#define L2 1
+#define L3 2
+#define SLC 3
+/* Retention policies. */
+#define KEEP 0
+#define STRM 1
+
+void
+prefetch_for_read_write (void *a)
+{
+ __pldx (PLD, L1, KEEP, a);
+ __pldx (PLD, L1, STRM, a);
+ __pldx (PLD, L2, KEEP, a);
+ __pldx (PLD, L2, STRM, a);
+ __pldx (PLD, L3, KEEP, a);
+ __pldx (PLD, L3, STRM, a);
+ __pldx (PLD, SLC, KEEP, a);
+ __pldx (PLD, SLC, STRM, a);
+ __pldx (PST, L1, KEEP, a);
+ __pldx (PST, L1, STRM, a);
+ __pldx (PST, L2, KEEP, a);
+ __pldx (PST, L2, STRM, a);
+ __pldx (PST, L3, KEEP, a);
+ __pldx (PST, L3, STRM, a);
+ __pldx (PST, SLC, KEEP, a);
+ __pldx (PST, SLC, STRM, a);
+}
+
+/* { dg-final { scan-assembler "prfm\tPLDL1KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDL1STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDL2KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDL2STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDL3KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDL3STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDSLCKEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLDSLCSTRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTL1KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTL1STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTL2KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTL2STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTL3KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTL3STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTSLCKEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPSTSLCSTRM, \\\[x\[0-9\]+\\\]" } } */
+
+void
+prefetch_simple (void *a)
+{
+ __pld (a);
+ __pli (a);
+}
+
+/* { dg-final { scan-assembler "prfm\tPLDL1KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLIL1KEEP, \\\[x\[0-9\]+\\\]" } } */
+
+void
+prefetch_instructions (void *a)
+{
+ __plix (L1, KEEP, a);
+ __plix (L1, STRM, a);
+ __plix (L2, KEEP, a);
+ __plix (L2, STRM, a);
+ __plix (L3, KEEP, a);
+ __plix (L3, STRM, a);
+ __plix (SLC, KEEP, a);
+ __plix (SLC, STRM, a);
+}
+
+/* { dg-final { scan-assembler "prfm\tPLIL1KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLIL1STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLIL2KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLIL2STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLIL3KEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLIL3STRM, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLISLCKEEP, \\\[x\[0-9\]+\\\]" } } */
+/* { dg-final { scan-assembler "prfm\tPLISLCSTRM, \\\[x\[0-9\]+\\\]" } } */
+
--- /dev/null
+/* Check that PRFM-related bounds checks are applied correctly. */
+/* { dg-do compile } */
+#include <arm_acle.h>
+
+/* Access kind specifiers. */
+#define KIND_LOW -1
+#define KIND_HIGH 2
+/* Cache levels. */
+#define LEVEL_LOW -1
+#define LEVEL_HIGH 4
+/* Retention policies. */
+#define POLICY_LOW -1
+#define POLICY_HIGH 2
+
+void
+data_rw_prefetch_bad_bounds (void *a)
+{
+ __builtin_aarch64_pldx (KIND_LOW, 0, 0, a); /* { dg-error {argument 1 must be a constant immediate in range \[0,1\]} } */
+ __builtin_aarch64_pldx (KIND_HIGH, 0, 0, a); /* { dg-error {argument 1 must be a constant immediate in range \[0,1\]} } */
+ __builtin_aarch64_pldx (0, LEVEL_LOW, 0, a); /* { dg-error {argument 2 must be a constant immediate in range \[0,3\]} } */
+ __builtin_aarch64_pldx (0, LEVEL_HIGH, 0, a); /* { dg-error {argument 2 must be a constant immediate in range \[0,3\]} } */
+ __builtin_aarch64_pldx (0, 0, POLICY_LOW, a); /* { dg-error {argument 3 must be a constant immediate in range \[0,1\]} } */
+ __builtin_aarch64_pldx (0, 0, POLICY_HIGH, a); /* { dg-error {argument 3 must be a constant immediate in range \[0,1\]} } */
+}
+
+void
+insn_prefetch_bad_bounds (void *a)
+{
+ __builtin_aarch64_plix (LEVEL_LOW, 0, a); /* { dg-error {argument 1 must be a constant immediate in range \[0,3\]} } */
+ __builtin_aarch64_plix (LEVEL_HIGH, 0, a); /* { dg-error {argument 1 must be a constant immediate in range \[0,3\]} } */
+ __builtin_aarch64_plix (0, POLICY_LOW, a); /* { dg-error {argument 2 must be a constant immediate in range \[0,1\]} } */
+ __builtin_aarch64_plix (0, POLICY_HIGH, a); /* { dg-error {argument 2 must be a constant immediate in range \[0,1\]} } */
+}
projects / thirdparty / gcc.git / commitdiff
