From 5ba6128602fdc18122e8f4c3b88e6200e7afd444 Mon Sep 17 00:00:00 2001 From: rguenth Date: Thu, 17 Oct 2019 11:48:45 +0000 Subject: [PATCH] 2019-10-17 Richard Biener * tree-vect-loop.c (needs_fold_left_reduction_p): Export. (vect_is_simple_reduction): Move all validity checks ... (vectorizable_reduction): ... here. Compute whether we need a fold-left reduction here. * tree-vect-patterns.c (vect_reassociating_reduction_p): Merge both overloads, check needs_fold_left_reduction_p directly. * tree-vectorizer.h (needs_fold_left_reduction_p): Declare. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@277100 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 10 +++ gcc/tree-vect-loop.c | 127 ++++++++++++++++++--------------------- gcc/tree-vect-patterns.c | 23 +++---- gcc/tree-vectorizer.h | 1 + 4 files changed, 80 insertions(+), 81 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index bdafc5839f3f..eebb890282bd 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,13 @@ +2019-10-17 Richard Biener + + * tree-vect-loop.c (needs_fold_left_reduction_p): Export. + (vect_is_simple_reduction): Move all validity checks ... + (vectorizable_reduction): ... here. Compute whether we + need a fold-left reduction here. + * tree-vect-patterns.c (vect_reassociating_reduction_p): Merge + both overloads, check needs_fold_left_reduction_p directly. + * tree-vectorizer.h (needs_fold_left_reduction_p): Declare. + 2019-10-17 Richard Biener * tree-ssa-pre.c (create_component_ref_by_pieces_1): Fix diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index d2ddc669176e..0ad32cec2e2a 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -2536,7 +2536,7 @@ report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg) on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer overflow must wrap. */ -static bool +bool needs_fold_left_reduction_p (tree type, tree_code code) { /* CHECKME: check for !flag_finite_math_only too? */ @@ -2888,13 +2888,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, op1 = gimple_assign_rhs2 (def_stmt); op2 = gimple_assign_rhs3 (def_stmt); } - else if (!commutative_tree_code (code) || !associative_tree_code (code)) - { - if (dump_enabled_p ()) - report_vect_op (MSG_MISSED_OPTIMIZATION, def_stmt, - "reduction: not commutative/associative: "); - return NULL; - } else if (get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS) { op1 = gimple_assign_rhs1 (def_stmt); @@ -2917,18 +2910,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, return NULL; } - /* Check whether it's ok to change the order of the computation. - Generally, when vectorizing a reduction we change the order of the - computation. This may change the behavior of the program in some - cases, so we need to check that this is ok. One exception is when - vectorizing an outer-loop: the inner-loop is executed sequentially, - and therefore vectorizing reductions in the inner-loop during - outer-loop vectorization is safe. */ - tree type = TREE_TYPE (gimple_assign_lhs (def_stmt)); - if (STMT_VINFO_REDUC_TYPE (phi_info) == TREE_CODE_REDUCTION - && needs_fold_left_reduction_p (type, code)) - STMT_VINFO_REDUC_TYPE (phi_info) = FOLD_LEFT_REDUCTION; - /* Reduction is safe. We're dealing with one of the following: 1) integer arithmetic and no trapv 2) floating point arithmetic, and special flags permit this optimization @@ -5633,7 +5614,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node, loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); enum tree_code code; - internal_fn reduc_fn; int op_type; enum vect_def_type dt, cond_reduc_dt = vect_unknown_def_type; stmt_vec_info cond_stmt_vinfo = NULL; @@ -5873,19 +5853,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node, operation in the reduction meta. */ STMT_VINFO_REDUC_IDX (reduc_info) = reduc_index; - /* When vectorizing a reduction chain w/o SLP the reduction PHI is not - directy used in stmt. */ - if (reduc_index == -1) - { - if (STMT_VINFO_REDUC_TYPE (phi_info) == FOLD_LEFT_REDUCTION) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "in-order reduction chain without SLP.\n"); - return false; - } - } - if (!(reduc_index == -1 || dts[reduc_index] == vect_reduction_def || dts[reduc_index] == vect_nested_cycle @@ -6048,17 +6015,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node, double_reduc = true; } - vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info); - if ((double_reduc || reduction_type != TREE_CODE_REDUCTION) - && ncopies > 1) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "multiple types in double reduction or condition " - "reduction.\n"); - return false; - } - /* 4.2. Check support for the epilog operation. If STMT represents a reduction pattern, then the type of the @@ -6094,38 +6050,75 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node, (and also the same tree-code) when generating the epilog code and when generating the code inside the loop. */ - enum tree_code orig_code; - if (orig_stmt_info - && (reduction_type == TREE_CODE_REDUCTION - || reduction_type == FOLD_LEFT_REDUCTION)) + vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info); + enum tree_code orig_code = ERROR_MARK; + if (reduction_type == CONST_COND_REDUCTION + || reduction_type == INTEGER_INDUC_COND_REDUCTION) { - /* This is a reduction pattern: get the vectype from the type of the - reduction variable, and get the tree-code from orig_stmt. */ - orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt); - gcc_assert (vectype_out); + /* For simple condition reductions, replace with the actual expression + we want to base our reduction around. */ + orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info); + gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR); } - else + else if (reduction_type == COND_REDUCTION) + orig_code = COND_EXPR; + else if (reduction_type == TREE_CODE_REDUCTION + || reduction_type == FOLD_LEFT_REDUCTION) { - /* Regular reduction: use the same vectype and tree-code as used for - the vector code inside the loop can be used for the epilog code. */ - orig_code = code; - - if (code == MINUS_EXPR) + if (orig_stmt_info) + orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt); + else + orig_code = code; + gcc_assert (vectype_out); + if (orig_code == MINUS_EXPR) orig_code = PLUS_EXPR; + } + STMT_VINFO_REDUC_CODE (reduc_info) = orig_code; - /* For simple condition reductions, replace with the actual expression - we want to base our reduction around. */ - if (reduction_type == CONST_COND_REDUCTION - || reduction_type == INTEGER_INDUC_COND_REDUCTION) + if (reduction_type == TREE_CODE_REDUCTION) + { + /* Check whether it's ok to change the order of the computation. + Generally, when vectorizing a reduction we change the order of the + computation. This may change the behavior of the program in some + cases, so we need to check that this is ok. One exception is when + vectorizing an outer-loop: the inner-loop is executed sequentially, + and therefore vectorizing reductions in the inner-loop during + outer-loop vectorization is safe. */ + if (needs_fold_left_reduction_p (scalar_type, orig_code)) + { + STMT_VINFO_REDUC_TYPE (reduc_info) + = reduction_type = FOLD_LEFT_REDUCTION; + /* When vectorizing a reduction chain w/o SLP the reduction PHI is not + directy used in stmt. */ + if (reduc_index == -1) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "in-order reduction chain without SLP.\n"); + return false; + } + } + else if (!commutative_tree_code (orig_code) + || !associative_tree_code (orig_code)) { - orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info); - gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR); + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "reduction: not commutative/associative"); + return false; } } - STMT_VINFO_REDUC_CODE (reduc_info) = orig_code; - reduc_fn = IFN_LAST; + if ((double_reduc || reduction_type != TREE_CODE_REDUCTION) + && ncopies > 1) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "multiple types in double reduction or condition " + "reduction or fold-left reduction.\n"); + return false; + } + internal_fn reduc_fn = IFN_LAST; if (reduction_type == TREE_CODE_REDUCTION || reduction_type == FOLD_LEFT_REDUCTION || reduction_type == INTEGER_INDUC_COND_REDUCTION diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 385482ffb988..5387563a15ed 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -833,19 +833,8 @@ vect_convert_output (stmt_vec_info stmt_info, tree type, gimple *pattern_stmt, /* Return true if STMT_VINFO describes a reduction for which reassociation is allowed. If STMT_INFO is part of a group, assume that it's part of a reduction chain and optimistically assume that all statements - except the last allow reassociation. */ - -static bool -vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo) -{ - if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) - return (STMT_VINFO_REDUC_TYPE (STMT_VINFO_REDUC_DEF (stmt_vinfo)) - != FOLD_LEFT_REDUCTION); - else - return REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL; -} - -/* As above, but also require it to have code CODE and to be a reduction + except the last allow reassociation. + Also require it to have code CODE and to be a reduction in the outermost loop. When returning true, store the operands in *OP0_OUT and *OP1_OUT. */ @@ -867,7 +856,13 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_info, tree_code code, if (loop && nested_in_vect_loop_p (loop, stmt_info)) return false; - if (!vect_reassociating_reduction_p (stmt_info)) + if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def) + { + if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)), + code)) + return false; + } + else if (REDUC_GROUP_FIRST_ELEMENT (stmt_info) == NULL) return false; *op0_out = gimple_assign_rhs1 (assign); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 291304fe95e1..559d78d4491c 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1632,6 +1632,7 @@ extern widest_int vect_iv_limit_for_full_masking (loop_vec_info loop_vinfo); /* Used in gimple-loop-interchange.c and tree-parloops.c. */ extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree, enum tree_code); +extern bool needs_fold_left_reduction_p (tree, tree_code); /* Drive for loop analysis stage. */ extern opt_loop_vec_info vect_analyze_loop (class loop *, loop_vec_info, -- 2.39.2