From: victork Date: Mon, 17 Sep 2007 09:37:31 +0000 (+0000) Subject: PR tree-optimization/33319 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a61b0edbf75b014cbe1c2c594bc801ed14c4f15f;p=thirdparty%2Fgcc.git PR tree-optimization/33319 * tree-vect-analyze.c (vect_same_range_drs): New. (vect_vfa_range_equal): New. (vect_is_duplicate_ddr): Removed. (vect_mark_for_runtime_alias_test): Do not perform marking when optimizing for size or max_param for alias checking is zero. Move the function before vect_analyze_data_ref_dependence. (vect_analyze_data_ref_dependence): Add call to vect_mark_for_runtime_alias_test in two cases when dependence is not clear. (vect_analyze_data_ref_dependences): Do not call to vect_mark_for_runtime_alias_test. (vect_prune_runtime_alias_test_list): New. (vect_analyze_loop): Add call to vect_prune_runtime_alias_test_list. * tree-vect-transform.c (vect_estimate_min_profitable_iters): Update vec_outside_cost. (vect_vfa_segment_size): More compact code, use TYPE_SIZE_UNIT. (vect_create_cond_for_alias_checks): Build the base address of data reference from DR_GROUP_FIRST_DR. (vect_loop_versioning): New. (vect_transform_loop): Add a call to vect_loop_versioning. Remove factored out code. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@128539 138bc75d-0d04-0410-961f-82ee72b054a4 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 1239d63c2ac8..2099869bbe2e 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,28 @@ +2007-09-17 Victor Kaplansky + + PR tree-optimization/33319 + * tree-vect-analyze.c (vect_same_range_drs): New. + (vect_vfa_range_equal): New. + (vect_is_duplicate_ddr): Removed. + (vect_mark_for_runtime_alias_test): Do not perform marking when + optimizing for size or max_param for alias checking is zero. + Move the function before vect_analyze_data_ref_dependence. + (vect_analyze_data_ref_dependence): Add call to + vect_mark_for_runtime_alias_test in two cases when dependence + is not clear. + (vect_analyze_data_ref_dependences): Do not call to + vect_mark_for_runtime_alias_test. + (vect_prune_runtime_alias_test_list): New. + (vect_analyze_loop): Add call to vect_prune_runtime_alias_test_list. + * tree-vect-transform.c (vect_estimate_min_profitable_iters): + Update vec_outside_cost. + (vect_vfa_segment_size): More compact code, use TYPE_SIZE_UNIT. + (vect_create_cond_for_alias_checks): Build the base address of data + reference from DR_GROUP_FIRST_DR. + (vect_loop_versioning): New. + (vect_transform_loop): Add a call to vect_loop_versioning. + Remove factored out code. + 2007-09-16 John David Anglin PR middle-end/33273 diff --git a/gcc/tree-vect-analyze.c b/gcc/tree-vect-analyze.c index c9f559be7cd3..86ca6b65ca47 100644 --- a/gcc/tree-vect-analyze.c +++ b/gcc/tree-vect-analyze.c @@ -1118,11 +1118,85 @@ vect_check_interleaving (struct data_reference *dra, } } +/* Check if data references pointed by DR_I and DR_J are same or + belong to same interleaving group. Return FALSE if drs are + different, otherwise return TRUE. */ + +static bool +vect_same_range_drs (data_reference_p dr_i, data_reference_p dr_j) +{ + tree stmt_i = DR_STMT (dr_i); + tree stmt_j = DR_STMT (dr_j); + + if (operand_equal_p (DR_REF (dr_i), DR_REF (dr_j), 0) + || (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_i)) + && DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_j)) + && (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_i)) + == DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_j))))) + return true; + else + return false; +} + +/* If address ranges represented by DDR_I and DDR_J are equal, + return TRUE, otherwise return FALSE. */ + +static bool +vect_vfa_range_equal (ddr_p ddr_i, ddr_p ddr_j) +{ + if ((vect_same_range_drs (DDR_A (ddr_i), DDR_A (ddr_j)) + && vect_same_range_drs (DDR_B (ddr_i), DDR_B (ddr_j))) + || (vect_same_range_drs (DDR_A (ddr_i), DDR_B (ddr_j)) + && vect_same_range_drs (DDR_B (ddr_i), DDR_A (ddr_j)))) + return true; + else + return false; +} + +/* Insert DDR into LOOP_VINFO list of ddrs that may alias and need to be + tested at run-time. Return TRUE if DDR was successfully inserted. + Return false if versioning is not supported. */ + +static bool +vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo) +{ + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + + if ((unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS) == 0) + return false; + + if (vect_print_dump_info (REPORT_DR_DETAILS)) + { + fprintf (vect_dump, "mark for run-time aliasing test between "); + print_generic_expr (vect_dump, DR_REF (DDR_A (ddr)), TDF_SLIM); + fprintf (vect_dump, " and "); + print_generic_expr (vect_dump, DR_REF (DDR_B (ddr)), TDF_SLIM); + } + + if (optimize_size) + { + if (vect_print_dump_info (REPORT_DR_DETAILS)) + fprintf (vect_dump, "versioning not supported when optimizing for size."); + return false; + } + + /* FORNOW: We don't support versioning with outer-loop vectorization. */ + if (loop->inner) + { + if (vect_print_dump_info (REPORT_DR_DETAILS)) + fprintf (vect_dump, "versioning not yet supported for outer-loops."); + return false; + } + + VEC_safe_push (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr); + return true; +} /* Function vect_analyze_data_ref_dependence. Return TRUE if there (might) exist a dependence between a memory-reference - DRA and a memory-reference DRB. */ + DRA and a memory-reference DRB. When versioning for alias may check a + dependence at run-time, return FALSE. */ static bool vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, @@ -1160,7 +1234,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, fprintf (vect_dump, " and "); print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM); } - return true; + /* Add to list of ddrs that need to be tested at run-time. */ + return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo); } if (DDR_NUM_DIST_VECTS (ddr) == 0) @@ -1172,7 +1247,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, fprintf (vect_dump, " and "); print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM); } - return true; + /* Add to list of ddrs that need to be tested at run-time. */ + return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo); } loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr)); @@ -1224,10 +1300,10 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, continue; } - if (vect_print_dump_info (REPORT_DR_DETAILS)) + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) { fprintf (vect_dump, - "versioning for alias required: possible dependence " + "not vectorized, possible dependence " "between data-refs "); print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM); fprintf (vect_dump, " and "); @@ -1240,88 +1316,6 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, return false; } -/* Return TRUE if DDR_NEW is already found in MAY_ALIAS_DDRS list. */ - -static bool -vect_is_duplicate_ddr (VEC (ddr_p, heap) * may_alias_ddrs, ddr_p ddr_new) -{ - unsigned i; - ddr_p ddr; - - for (i = 0; VEC_iterate (ddr_p, may_alias_ddrs, i, ddr); i++) - { - tree dref_A_i, dref_B_i, dref_A_j, dref_B_j; - - dref_A_i = DR_REF (DDR_A (ddr)); - dref_B_i = DR_REF (DDR_B (ddr)); - dref_A_j = DR_REF (DDR_A (ddr_new)); - dref_B_j = DR_REF (DDR_B (ddr_new)); - - if ((operand_equal_p (dref_A_i, dref_A_j, 0) - && operand_equal_p (dref_B_i, dref_B_j, 0)) - || (operand_equal_p (dref_A_i, dref_B_j, 0) - && operand_equal_p (dref_B_i, dref_A_j, 0))) - { - if (vect_print_dump_info (REPORT_DR_DETAILS)) - { - fprintf (vect_dump, "found same pair of data references "); - print_generic_expr (vect_dump, dref_A_i, TDF_SLIM); - fprintf (vect_dump, " and "); - print_generic_expr (vect_dump, dref_B_i, TDF_SLIM); - } - return true; - } - } - return false; -} - -/* Save DDR in LOOP_VINFO list of ddrs that may alias and need to be - tested at run-time. Returns false if number of run-time checks - inserted by vectorizer is greater than maximum defined by - PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS. */ -static bool -vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo) -{ - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - - if (vect_print_dump_info (REPORT_DR_DETAILS)) - { - fprintf (vect_dump, "mark for run-time aliasing test between "); - print_generic_expr (vect_dump, DR_REF (DDR_A (ddr)), TDF_SLIM); - fprintf (vect_dump, " and "); - print_generic_expr (vect_dump, DR_REF (DDR_B (ddr)), TDF_SLIM); - } - - /* FORNOW: We don't support versioning with outer-loop vectorization. */ - if (loop->inner) - { - if (vect_print_dump_info (REPORT_DR_DETAILS)) - fprintf (vect_dump, "versioning not yet supported for outer-loops."); - return false; - } - - /* Do not add to the list duplicate ddrs. */ - if (vect_is_duplicate_ddr (LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr)) - return true; - - if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)) - >= (unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS)) - { - if (vect_print_dump_info (REPORT_DR_DETAILS)) - { - fprintf (vect_dump, - "disable versioning for alias - max number of generated " - "checks exceeded."); - } - - VEC_truncate (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), 0); - - return false; - } - VEC_safe_push (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr); - return true; -} - /* Function vect_analyze_data_ref_dependences. Examine all the data references in the loop, and make sure there do not @@ -1339,11 +1333,7 @@ vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo) for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++) if (vect_analyze_data_ref_dependence (ddr, loop_vinfo)) - { - /* Add to list of ddrs that need to be tested at run-time. */ - if (!vect_mark_for_runtime_alias_test (ddr, loop_vinfo)) return false; - } return true; } @@ -2381,6 +2371,77 @@ vect_analyze_data_ref_accesses (loop_vec_info loop_vinfo) return true; } +/* Function vect_prune_runtime_alias_test_list. + + Prune a list of ddrs to be tested at run-time by versioning for alias. + Return FALSE if resulting list of ddrs is longer then allowed by + PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS, otherwise return TRUE. */ + +static bool +vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) +{ + VEC (ddr_p, heap) * ddrs = + LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo); + unsigned i, j; + + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "=== vect_prune_runtime_alias_test_list ==="); + + for (i = 0; i < VEC_length (ddr_p, ddrs); ) + { + bool found; + ddr_p ddr_i; + + ddr_i = VEC_index (ddr_p, ddrs, i); + found = false; + + for (j = 0; j < i; j++) + { + ddr_p ddr_j = VEC_index (ddr_p, ddrs, j); + + if (vect_vfa_range_equal (ddr_i, ddr_j)) + { + if (vect_print_dump_info (REPORT_DR_DETAILS)) + { + fprintf (vect_dump, "found equal ranges "); + print_generic_expr (vect_dump, DR_REF (DDR_A (ddr_i)), TDF_SLIM); + fprintf (vect_dump, ", "); + print_generic_expr (vect_dump, DR_REF (DDR_B (ddr_i)), TDF_SLIM); + fprintf (vect_dump, " and "); + print_generic_expr (vect_dump, DR_REF (DDR_A (ddr_j)), TDF_SLIM); + fprintf (vect_dump, ", "); + print_generic_expr (vect_dump, DR_REF (DDR_B (ddr_j)), TDF_SLIM); + } + found = true; + break; + } + } + + if (found) + { + VEC_ordered_remove (ddr_p, ddrs, i); + continue; + } + i++; + } + + if (VEC_length (ddr_p, ddrs) > + (unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS)) + { + if (vect_print_dump_info (REPORT_DR_DETAILS)) + { + fprintf (vect_dump, + "disable versioning for alias - max number of generated " + "checks exceeded."); + } + + VEC_truncate (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), 0); + + return false; + } + + return true; +} /* Recursively free the memory allocated for the SLP tree rooted at NODE. */ @@ -4231,6 +4292,19 @@ vect_analyze_loop (struct loop *loop) return NULL; } + /* Prune the list of ddrs to be tested at run-time by versioning for alias. + It is important to call pruning after vect_analyze_data_ref_accesses, + since we use grouping information gathered by interleaving analysis. */ + ok = vect_prune_runtime_alias_test_list (loop_vinfo); + if (!ok) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "too long list of versioning for alias " + "run-time tests."); + destroy_loop_vec_info (loop_vinfo, true); + return NULL; + } + /* Check the SLP opportunities in the loop, analyze and build SLP trees. */ ok = vect_analyze_slp (loop_vinfo); if (ok) diff --git a/gcc/tree-vect-transform.c b/gcc/tree-vect-transform.c index df4bd08f4cd1..f02d912b29f6 100644 --- a/gcc/tree-vect-transform.c +++ b/gcc/tree-vect-transform.c @@ -137,15 +137,32 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo) return 0; } - /* Requires loop versioning tests to handle misalignment. - FIXME: Make cost depend on number of stmts in may_misalign list. */ + /* Requires loop versioning tests to handle misalignment. */ if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))) { - vec_outside_cost += TARG_COND_TAKEN_BRANCH_COST; + /* FIXME: Make cost depend on complexity of individual check. */ + vec_outside_cost += + VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)); + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "cost model: Adding cost of checks for loop " + "versioning to treat misalignment.\n"); + } + + if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))) + { + /* FIXME: Make cost depend on complexity of individual check. */ + vec_outside_cost += + VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)); if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "cost model: Adding cost of checks for loop " - "versioning.\n"); + "versioning aliasing.\n"); + } + + if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)) + || VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))) + { + vec_outside_cost += TARG_COND_TAKEN_BRANCH_COST; } /* Count statements in scalar loop. Using this as scalar cost for a single @@ -6864,31 +6881,18 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo, static tree vect_vfa_segment_size (struct data_reference *dr, tree vect_factor) { - tree segment_length; + tree segment_length = fold_build2 (MULT_EXPR, integer_type_node, + DR_STEP (dr), vect_factor); if (vect_supportable_dr_alignment (dr) == dr_explicit_realign_optimized) { - tree vector_size = - build_int_cst (integer_type_node, - GET_MODE_SIZE (TYPE_MODE (STMT_VINFO_VECTYPE - (vinfo_for_stmt (DR_STMT (dr)))))); + tree vector_size = TYPE_SIZE_UNIT + (STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr)))); - segment_length = - fold_convert (sizetype, - fold_build2 (PLUS_EXPR, integer_type_node, - fold_build2 (MULT_EXPR, integer_type_node, DR_STEP (dr), - vect_factor), - vector_size)); + segment_length = fold_build2 (PLUS_EXPR, integer_type_node, + segment_length, vector_size); } - else - { - segment_length = - fold_convert (sizetype, - fold_build2 (MULT_EXPR, integer_type_node, DR_STEP (dr), - vect_factor)); - } - - return segment_length; + return fold_convert (sizetype, segment_length); } /* Function vect_create_cond_for_alias_checks. @@ -6907,6 +6911,8 @@ vect_vfa_segment_size (struct data_reference *dr, tree vect_factor) COND_EXPR - conditional expression. COND_EXPR_STMT_LIST - statements needed to construct the conditional expression. + + The returned value is the conditional expression to be used in the if statement that controls which version of the loop gets executed at runtime. */ @@ -6940,26 +6946,47 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo, for (i = 0; VEC_iterate (ddr_p, may_alias_ddrs, i, ddr); i++) { - tree stmt_a = DR_STMT (DDR_A (ddr)); - tree stmt_b = DR_STMT (DDR_B (ddr)); + struct data_reference *dr_a, *dr_b; + tree dr_group_first_a, dr_group_first_b; + tree addr_base_a, addr_base_b; + tree segment_length_a, segment_length_b; + tree stmt_a, stmt_b; - tree addr_base_a = + dr_a = DDR_A (ddr); + stmt_a = DR_STMT (DDR_A (ddr)); + dr_group_first_a = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_a)); + if (dr_group_first_a) + { + stmt_a = dr_group_first_a; + dr_a = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_a)); + } + + dr_b = DDR_B (ddr); + stmt_b = DR_STMT (DDR_B (ddr)); + dr_group_first_b = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_b)); + if (dr_group_first_b) + { + stmt_b = dr_group_first_b; + dr_b = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_b)); + } + + addr_base_a = vect_create_addr_base_for_vector_ref (stmt_a, cond_expr_stmt_list, NULL_TREE, loop); - tree addr_base_b = + addr_base_b = vect_create_addr_base_for_vector_ref (stmt_b, cond_expr_stmt_list, NULL_TREE, loop); - tree segment_length_a = vect_vfa_segment_size (DDR_A (ddr), vect_factor); - tree segment_length_b = vect_vfa_segment_size (DDR_B (ddr), vect_factor); + segment_length_a = vect_vfa_segment_size (dr_a, vect_factor); + segment_length_b = vect_vfa_segment_size (dr_b, vect_factor); if (vect_print_dump_info (REPORT_DR_DETAILS)) { fprintf (vect_dump, "create runtime check for data references "); - print_generic_expr (vect_dump, DR_REF (DDR_A (ddr)), TDF_SLIM); + print_generic_expr (vect_dump, DR_REF (dr_a), TDF_SLIM); fprintf (vect_dump, " and "); - print_generic_expr (vect_dump, DR_REF (DDR_B (ddr)), TDF_SLIM); + print_generic_expr (vect_dump, DR_REF (dr_b), TDF_SLIM); } @@ -6988,6 +7015,91 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo, } +/* Function vect_loop_versioning. + + If the loop has data references that may or may not be aligned or/and + has data reference relations whose independence was not proven then + two versions of the loop need to be generated, one which is vectorized + and one which isn't. A test is then generated to control which of the + loops is executed. The test checks for the alignment of all of the + data references that may or may not be aligned. An additional + sequence of runtime tests is generated for each pairs of DDRs whose + independence was not proven. The vectorized version of loop is + executed only if both alias and alignment tests are passed. */ + +static void +vect_loop_versioning (loop_vec_info loop_vinfo) +{ + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + struct loop *nloop; + tree cond_expr = NULL_TREE; + tree cond_expr_stmt_list = NULL_TREE; + basic_block condition_bb; + block_stmt_iterator cond_exp_bsi; + basic_block merge_bb; + basic_block new_exit_bb; + edge new_exit_e, e; + tree orig_phi, new_phi, arg; + unsigned prob = 4 * REG_BR_PROB_BASE / 5; + tree gimplify_stmt_list; + + if (!VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)) + && !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))) + return; + + if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))) + cond_expr = + vect_create_cond_for_align_checks (loop_vinfo, &cond_expr_stmt_list); + + if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))) + vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr, &cond_expr_stmt_list); + + cond_expr = + fold_build2 (NE_EXPR, boolean_type_node, cond_expr, integer_zero_node); + cond_expr = + force_gimple_operand (cond_expr, &gimplify_stmt_list, true, + NULL_TREE); + append_to_statement_list (gimplify_stmt_list, &cond_expr_stmt_list); + + initialize_original_copy_tables (); + nloop = loop_version (loop, cond_expr, &condition_bb, + prob, prob, REG_BR_PROB_BASE - prob, true); + free_original_copy_tables(); + + /* Loop versioning violates an assumption we try to maintain during + vectorization - that the loop exit block has a single predecessor. + After versioning, the exit block of both loop versions is the same + basic block (i.e. it has two predecessors). Just in order to simplify + following transformations in the vectorizer, we fix this situation + here by adding a new (empty) block on the exit-edge of the loop, + with the proper loop-exit phis to maintain loop-closed-form. */ + + merge_bb = single_exit (loop)->dest; + gcc_assert (EDGE_COUNT (merge_bb->preds) == 2); + new_exit_bb = split_edge (single_exit (loop)); + new_exit_e = single_exit (loop); + e = EDGE_SUCC (new_exit_bb, 0); + + for (orig_phi = phi_nodes (merge_bb); orig_phi; + orig_phi = PHI_CHAIN (orig_phi)) + { + new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)), + new_exit_bb); + arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e); + add_phi_arg (new_phi, arg, new_exit_e); + SET_PHI_ARG_DEF (orig_phi, e->dest_idx, PHI_RESULT (new_phi)); + } + + /* End loop-exit-fixes after versioning. */ + + update_ssa (TODO_update_ssa); + if (cond_expr_stmt_list) + { + cond_exp_bsi = bsi_last (condition_bb); + bsi_insert_before (&cond_exp_bsi, cond_expr_stmt_list, BSI_SAME_STMT); + } +} + /* Remove a group of stores (for SLP or interleaving), free their stmt_vec_info. */ @@ -7096,7 +7208,6 @@ vect_schedule_slp (loop_vec_info loop_vinfo, unsigned int nunits) return is_store; } - /* Function vect_transform_loop. The analysis phase has determined that the loop is vectorizable. @@ -7119,82 +7230,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "=== vec_transform_loop ==="); - - /* If the loop has data references that may or may not be aligned or/and - has data reference relations whose independence was not proven then - two versions of the loop need to be generated, one which is vectorized - and one which isn't. A test is then generated to control which of the - loops is executed. The test checks for the alignment of all of the - data references that may or may not be aligned. An additional - sequence of runtime tests is generated for each pairs of DDRs whose - independence was not proven. The vectorized version of loop is - executed only if both alias and alignment tests are passed. */ - - if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)) - || VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))) - { - struct loop *nloop; - tree cond_expr = NULL_TREE; - tree cond_expr_stmt_list = NULL_TREE; - basic_block condition_bb; - block_stmt_iterator cond_exp_bsi; - basic_block merge_bb; - basic_block new_exit_bb; - edge new_exit_e, e; - tree orig_phi, new_phi, arg; - unsigned prob = 4 * REG_BR_PROB_BASE / 5; - tree gimplify_stmt_list; - - if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))) - cond_expr = - vect_create_cond_for_align_checks (loop_vinfo, &cond_expr_stmt_list); - - if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))) - vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr, - &cond_expr_stmt_list); - - cond_expr = - fold_build2 (NE_EXPR, boolean_type_node, cond_expr, integer_zero_node); - cond_expr = - force_gimple_operand (cond_expr, &gimplify_stmt_list, true, - NULL_TREE); - append_to_statement_list (gimplify_stmt_list, &cond_expr_stmt_list); - - initialize_original_copy_tables (); - nloop = loop_version (loop, cond_expr, &condition_bb, - prob, prob, REG_BR_PROB_BASE - prob, true); - free_original_copy_tables(); - - /** Loop versioning violates an assumption we try to maintain during - vectorization - that the loop exit block has a single predecessor. - After versioning, the exit block of both loop versions is the same - basic block (i.e. it has two predecessors). Just in order to simplify - following transformations in the vectorizer, we fix this situation - here by adding a new (empty) block on the exit-edge of the loop, - with the proper loop-exit phis to maintain loop-closed-form. **/ - - merge_bb = single_exit (loop)->dest; - gcc_assert (EDGE_COUNT (merge_bb->preds) == 2); - new_exit_bb = split_edge (single_exit (loop)); - new_exit_e = single_exit (loop); - e = EDGE_SUCC (new_exit_bb, 0); - - for (orig_phi = phi_nodes (merge_bb); orig_phi; - orig_phi = PHI_CHAIN (orig_phi)) - { - new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)), - new_exit_bb); - arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e); - add_phi_arg (new_phi, arg, new_exit_e); - SET_PHI_ARG_DEF (orig_phi, e->dest_idx, PHI_RESULT (new_phi)); - } - - /** end loop-exit-fixes after versioning **/ - - update_ssa (TODO_update_ssa); - cond_exp_bsi = bsi_last (condition_bb); - bsi_insert_before (&cond_exp_bsi, cond_expr_stmt_list, BSI_SAME_STMT); - } + vect_loop_versioning (loop_vinfo); /* CHECKME: we wouldn't need this if we called update_ssa once for all loops. */