From: Richard Sandiford Date: Thu, 19 Sep 2019 13:33:55 +0000 (+0000) Subject: Enforce correct COND_EXPR order for EXTRACT_LAST_REDUCTION X-Git-Tag: misc/cutover-git~2612 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c449d3ae28ff4e133114fb67dbf7dcc7a95ca5d5;p=thirdparty%2Fgcc.git Enforce correct COND_EXPR order for EXTRACT_LAST_REDUCTION For conditional reductions, the "then" value needs to be the candidate value calculated by this iteration while the "else" value needs to be the result carried over from previous iterations. If the COND_EXPR is the other way around, we need to swap it. 2019-09-19 Richard Sandiford gcc/ * tree-vectorizer.h (vectorizable_condition): Take an int reduction index instead of a boolean flag. * tree-vect-stmts.c (vectorizable_condition): Likewise. Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION reductions if the reduction accumulator is the "then" rather than the "else" value. (vect_analyze_stmt): Update call accordingly. (vect_transform_stmt): Likewise. * tree-vect-loop.c (vectorizable_reduction): Likewise, asserting that the index is > 0. From-SVN: r275962 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index cbddb904bfff..506e93d757ca 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,16 @@ +2019-09-19 Richard Sandiford + + * tree-vectorizer.h (vectorizable_condition): Take an int + reduction index instead of a boolean flag. + * tree-vect-stmts.c (vectorizable_condition): Likewise. + Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION + reductions if the reduction accumulator is the "then" rather + than the "else" value. + (vect_analyze_stmt): Update call accordingly. + (vect_transform_stmt): Likewise. + * tree-vect-loop.c (vectorizable_reduction): Likewise, + asserting that the index is > 0. + 2019-09-19 Martin Liska * ipa-icf.c (sort_sem_items_by_decl_uid): Simplify comparator. diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 5ee4ee3063f0..aec44622e03c 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -6659,8 +6659,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, { /* Only call during the analysis stage, otherwise we'll lose STMT_VINFO_TYPE. */ + gcc_assert (reduc_index > 0); if (!vec_stmt && !vectorizable_condition (stmt_info, gsi, NULL, - true, NULL, cost_vec)) + reduc_index, NULL, cost_vec)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -7113,9 +7114,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, if (reduction_type == EXTRACT_LAST_REDUCTION) { - gcc_assert (!slp_node); + gcc_assert (!slp_node && reduc_index > 0); return vectorizable_condition (stmt_info, gsi, vec_stmt, - true, NULL, NULL); + reduc_index, NULL, NULL); } /* Create the destination vector */ @@ -7145,9 +7146,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, { if (code == COND_EXPR) { - gcc_assert (!slp_node); + gcc_assert (!slp_node && reduc_index > 0); vectorizable_condition (stmt_info, gsi, vec_stmt, - true, NULL, NULL); + reduc_index, NULL, NULL); break; } if (code == LSHIFT_EXPR diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 7f5f1c3fafaf..0636e5c9ffa0 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -9778,7 +9778,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, bool vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, bool for_reduction, + stmt_vec_info *vec_stmt, int reduc_index, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { vec_info *vinfo = stmt_info->vinfo; @@ -9807,6 +9807,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vec vec_oprnds3 = vNULL; tree vec_cmp_type; bool masked = false; + bool for_reduction = (reduc_index > 0); if (for_reduction && STMT_SLP_TYPE (stmt_info)) return false; @@ -9888,6 +9889,29 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, cond_expr1 = TREE_OPERAND (cond_expr, 1); } + /* For conditional reductions, the "then" value needs to be the candidate + value calculated by this iteration while the "else" value needs to be + the result carried over from previous iterations. If the COND_EXPR + is the other way around, we need to swap it. */ + bool must_invert_cmp_result = false; + if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1) + { + if (masked) + must_invert_cmp_result = true; + else + { + bool honor_nans = HONOR_NANS (TREE_TYPE (cond_expr0)); + tree_code new_code = invert_tree_comparison (cond_code, honor_nans); + if (new_code == ERROR_MARK) + must_invert_cmp_result = true; + else + cond_code = new_code; + } + /* Make sure we don't accidentally use the old condition. */ + cond_expr = NULL_TREE; + std::swap (then_clause, else_clause); + } + if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype)) { /* Boolean values may have another representation in vectors @@ -10102,6 +10126,15 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vect_finish_stmt_generation (stmt_info, new_stmt, gsi); vec_compare = vec_compare_name; } + if (must_invert_cmp_result) + { + tree vec_compare_name = make_ssa_name (vec_cmp_type); + gassign *new_stmt = gimple_build_assign (vec_compare_name, + BIT_NOT_EXPR, + vec_compare); + vect_finish_stmt_generation (stmt_info, new_stmt, gsi); + vec_compare = vec_compare_name; + } gcall *new_stmt = gimple_build_call_internal (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, vec_then_clause); @@ -10635,7 +10668,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize, node_instance, cost_vec) || vectorizable_induction (stmt_info, NULL, NULL, node, cost_vec) || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (stmt_info, NULL, NULL, false, node, + || vectorizable_condition (stmt_info, NULL, NULL, 0, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, cost_vec)); @@ -10654,7 +10687,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize, || vectorizable_load (stmt_info, NULL, NULL, node, node_instance, cost_vec) || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (stmt_info, NULL, NULL, false, node, + || vectorizable_condition (stmt_info, NULL, NULL, 0, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, cost_vec)); @@ -10759,7 +10792,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, break; case condition_vec_info_type: - done = vectorizable_condition (stmt_info, gsi, &vec_stmt, false, + done = vectorizable_condition (stmt_info, gsi, &vec_stmt, 0, slp_node, NULL); gcc_assert (done); break; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index ac6e89907a30..aef0ff8663c7 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1534,7 +1534,7 @@ extern void vect_remove_stores (stmt_vec_info); extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *); extern bool vectorizable_condition (stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, bool, slp_tree, + stmt_vec_info *, int, slp_tree, stmt_vector_for_cost *); extern bool vectorizable_shift (stmt_vec_info, gimple_stmt_iterator *, stmt_vec_info *, slp_tree,