/* Vectorizer
- Copyright (C) 2003-2019 Free Software Foundation, Inc.
+ Copyright (C) 2003-2020 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
#include "tree-data-ref.h"
#include "tree-hash-traits.h"
#include "target.h"
+#include <utility>
/* Used for naming of new temporaries. */
enum vect_var_kind {
vec<slp_tree> children;
/* A group of scalar stmts to be vectorized together. */
vec<stmt_vec_info> stmts;
+ /* A group of scalar operands to be vectorized together. */
+ vec<tree> ops;
/* Load permutation relative to the stores, NULL if there is no
permutation. */
vec<unsigned> load_permutation;
/* The root of SLP tree. */
slp_tree root;
+ /* For vector constructors, the constructor stmt that the SLP tree is built
+ from, NULL otherwise. */
+ stmt_vec_info root_stmt;
+
/* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */
unsigned int group_size;
#define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size
#define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
#define SLP_INSTANCE_LOADS(S) (S)->loads
+#define SLP_INSTANCE_ROOT_STMT(S) (S)->root_stmt
#define SLP_TREE_CHILDREN(S) (S)->children
#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
+#define SLP_TREE_SCALAR_OPS(S) (S)->ops
#define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts
#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
/* Vectorizer state common between loop and basic-block vectorization. */
class vec_info {
public:
+ typedef hash_set<int_hash<machine_mode, E_VOIDmode, E_BLKmode> > mode_set;
enum vec_kind { bb, loop };
vec_info (vec_kind, void *, vec_info_shared *);
/* Cost data used by the target cost model. */
void *target_cost_data;
+ /* The set of vector modes used in the vectorized region. */
+ mode_set used_vector_modes;
+
+ /* The argument we should pass to related_vector_mode when looking up
+ the vector mode for a scalar mode, or VOIDmode if we haven't yet
+ made any decisions about which vector modes to use. */
+ machine_mode vector_mode;
+
private:
stmt_vec_info new_stmt_vec_info (gimple *stmt);
void set_vinfo_for_stmt (gimple *, stmt_vec_info);
typedef auto_vec<rgroup_masks> vec_loop_masks;
+typedef auto_vec<std::pair<data_reference*, tree> > drs_init_vec;
+
/*-----------------------------------------------------------------*/
/* Info on vectorized loops. */
/*-----------------------------------------------------------------*/
/* Threshold of number of iterations below which vectorization will not be
performed. It is calculated from MIN_PROFITABLE_ITERS and
- PARAM_MIN_VECT_LOOP_BOUND. */
+ param_min_vect_loop_bound. */
unsigned int th;
/* When applying loop versioning, the vector form should only be used
/* Cost of a single scalar iteration. */
int single_scalar_iteration_cost;
+ /* The cost of the vector prologue and epilogue, including peeled
+ iterations and set-up code. */
+ int vec_outside_cost;
+
+ /* The cost of the vector loop body. */
+ int vec_inside_cost;
+
/* Is the loop vectorizable? */
bool vectorizable;
this points to the original vectorized loop. Otherwise NULL. */
_loop_vec_info *orig_loop_info;
+ /* Used to store loop_vec_infos of epilogues of this loop during
+ analysis. */
+ vec<_loop_vec_info *> epilogue_vinfos;
+
} *loop_vec_info;
/* Access Functions. */
/* The vector input type relevant for reduction vectorization. */
tree reduc_vectype_in;
+ /* The vector type for performing the actual reduction. */
+ tree reduc_vectype;
+
/* Whether we force a single cycle PHI during reduction vectorization. */
bool force_single_cycle;
unsigned int operation_precision;
signop operation_sign;
+ /* If the statement produces a boolean result, this value describes
+ how we should choose the associated vector type. The possible
+ values are:
+
+ - an integer precision N if we should use the vector mask type
+ associated with N-bit integers. This is only used if all relevant
+ input booleans also want the vector mask type for N-bit integers,
+ or if we can convert them into that form by pattern-matching.
+
+ - ~0U if we considered choosing a vector mask type but decided
+ to treat the boolean as a normal integer type instead.
+
+ - 0 otherwise. This means either that the operation isn't one that
+ could have a vector mask type (and so should have a normal vector
+ type instead) or that we simply haven't made a choice either way. */
+ unsigned int mask_precision;
+
/* True if this is only suitable for SLP vectorization. */
bool slp_vect_only_p;
};
#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
+#define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
&& (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
}
+/* Return true if STMT_INFO should produce a vector mask type rather than
+ a normal nonmask type. */
+
+static inline bool
+vect_use_mask_type_p (stmt_vec_info stmt_info)
+{
+ return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
+}
+
/* Return TRUE if a statement represented by STMT_INFO is a part of a
pattern. */
static inline void
vect_update_max_nunits (poly_uint64 *max_nunits, poly_uint64 nunits)
{
- /* All unit counts have the form current_vector_size * X for some
+ /* All unit counts have the form vec_info::vector_size * X for some
rational X, so two unit sizes must have a common multiple.
Everything is a multiple of the initial value of 1. */
*max_nunits = force_common_multiple (*max_nunits, nunits);
return tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr_info->dr))));
}
+/* Return true if LOOP_VINFO requires a runtime check for whether the
+ vector loop is profitable. */
+
+inline bool
+vect_apply_runtime_profitability_check_p (loop_vec_info loop_vinfo)
+{
+ unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
+ return (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && th >= vect_vf_for_cost (loop_vinfo));
+}
+
/* Source location + hotness information. */
extern dump_user_location_t vect_location;
class loop *, edge);
class loop *vect_loop_versioning (loop_vec_info);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
- tree *, tree *, tree *, int, bool, bool);
+ tree *, tree *, tree *, int, bool, bool,
+ tree *, drs_init_vec &);
extern void vect_prepare_for_masked_peels (loop_vec_info);
extern dump_user_location_t find_loop_location (class loop *);
extern bool vect_can_advance_ivs_p (loop_vec_info);
+extern void vect_update_inits_of_drs (loop_vec_info, tree, tree_code);
/* In tree-vect-stmts.c. */
-extern poly_uint64 current_vector_size;
-extern tree get_vectype_for_scalar_type (vec_info *, tree);
-extern tree get_vectype_for_scalar_type_and_size (tree, poly_uint64);
-extern tree get_mask_type_for_scalar_type (vec_info *, tree);
+extern tree get_related_vectype_for_scalar_type (machine_mode, tree,
+ poly_uint64 = 0);
+extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0);
+extern tree get_vectype_for_scalar_type (vec_info *, tree, slp_tree);
+extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0);
extern tree get_same_sized_vectype (tree, tree);
+extern bool vect_chooses_same_modes_p (vec_info *, machine_mode);
extern bool vect_get_loop_mask_type (loop_vec_info);
extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
stmt_vec_info * = NULL, gimple ** = NULL);
enum tree_code *, int *,
vec<tree> *);
extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
- enum tree_code *,
- int *, vec<tree> *);
+ enum tree_code *, int *,
+ vec<tree> *);
extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
enum vect_cost_for_stmt, stmt_vec_info,
int, enum vect_cost_model_location);
extern bool vect_transform_stmt (stmt_vec_info, gimple_stmt_iterator *,
slp_tree, slp_instance);
extern void vect_remove_stores (stmt_vec_info);
+extern bool vect_nop_conversion_p (stmt_vec_info);
extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree,
slp_instance, stmt_vector_for_cost *);
extern void vect_get_load_cost (stmt_vec_info, int, bool,
extern gcall *vect_gen_while (tree, tree, tree);
extern tree vect_gen_while_not (gimple_seq *, tree, tree, tree);
extern opt_result vect_get_vector_types_for_stmt (stmt_vec_info, tree *,
- tree *);
-extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info);
+ tree *, unsigned int = 0);
+extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, unsigned int = 0);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance);
extern opt_result vect_analyze_data_ref_accesses (vec_info *);
extern opt_result vect_prune_runtime_alias_test_list (loop_vec_info);
-extern bool vect_gather_scatter_fn_p (bool, bool, tree, tree, unsigned int,
- signop, int, internal_fn *, tree *);
+extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
+ tree, int, internal_fn *, tree *);
extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
gather_scatter_info *);
extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
/* In tree-vect-loop.c. */
extern widest_int vect_iv_limit_for_full_masking (loop_vec_info loop_vinfo);
+/* Used in tree-vect-loop-manip.c */
+extern void determine_peel_for_niter (loop_vec_info);
/* Used in gimple-loop-interchange.c and tree-parloops.c. */
extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
enum tree_code);
extern bool needs_fold_left_reduction_p (tree, tree_code);
/* Drive for loop analysis stage. */
-extern opt_loop_vec_info vect_analyze_loop (class loop *,
- loop_vec_info,
- vec_info_shared *);
+extern opt_loop_vec_info vect_analyze_loop (class loop *, vec_info_shared *);
extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL);
extern void vect_gen_vector_loop_niters (loop_vec_info, tree, tree *,
tree *, bool);
-extern tree vect_halve_mask_nunits (tree);
-extern tree vect_double_mask_nunits (tree);
+extern tree vect_halve_mask_nunits (tree, machine_mode);
+extern tree vect_double_mask_nunits (tree, machine_mode);
extern void vect_record_loop_mask (loop_vec_info, vec_loop_masks *,
unsigned int, tree, tree);
extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *,
extern opt_result vect_analyze_slp (vec_info *, unsigned);
extern bool vect_make_slp_decision (loop_vec_info);
extern void vect_detect_hybrid_slp (loop_vec_info);
-extern void vect_get_slp_defs (vec<tree> , slp_tree, vec<vec<tree> > *);
+extern void vect_get_slp_defs (slp_tree, vec<vec<tree> > *, unsigned n = -1U);
extern bool vect_slp_bb (basic_block);
extern stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree);
extern bool is_simple_and_all_uses_invariant (stmt_vec_info, loop_vec_info);
-extern bool can_duplicate_and_interleave_p (unsigned int, machine_mode,
+extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree,
unsigned int * = NULL,
tree * = NULL, tree * = NULL);
-extern void duplicate_and_interleave (gimple_seq *, tree, vec<tree>,
- unsigned int, vec<tree> &);
+extern void duplicate_and_interleave (vec_info *, gimple_seq *, tree,
+ vec<tree>, unsigned int, vec<tree> &);
extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info);
/* In tree-vect-patterns.c. */
projects / thirdparty / gcc.git / blobdiff