From: liuhongt Date: Tue, 7 Jul 2026 03:14:01 +0000 (-0700) Subject: i386: Prefer shorter fold-left reduction chains X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=3f2b3eced0f8278fa335b1337a98b96df3ea062e;p=thirdparty%2Fgcc.git i386: Prefer shorter fold-left reduction chains Fold-left FP reductions are lowered to scalar operations on x86. In mixed-width loops the per-statement costs can therefore make a wider vector mode look better even though it lengthens the serial reduction chain. Count the fold-left reduction lanes from reduction groups and chains during finish_cost and use that as an extra loop-candidate preference. Do not let emulated sub-SSE modes win on this preference alone. Dump the lane count when present. gcc/ChangeLog: * config/i386/i386.cc (ix86_vector_costs): Add better_fold_left_reduc_than_p and m_num_fold_left_reduc_lanes. (ix86_vector_costs::ix86_vector_costs): Initialize it. (ix86_vector_costs::finish_cost): Count and dump fold-left reduction lanes. (ix86_vector_costs::better_fold_left_reduc_than_p): New function. (ix86_vector_costs::better_main_loop_than_p): Use it. (ix86_vector_costs::better_epilogue_loop_than_p): Likewise. gcc/testsuite/ChangeLog: * gcc.target/i386/fold-left-reduc-cost.c: New test. * gcc.target/i386/fold-left-reduc-chain-cost.c: New test. * gcc.target/i386/vect-epilogues-6.c: Update expected vector size. * gcc.target/i386/vect-epilogues-7.c: Likewise. * gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c: Likewise. --- diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index 804f35b2973..33e5e063b99 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -26452,6 +26452,7 @@ public: private: + bool better_fold_left_reduc_than_p (const vector_costs *) const; /* Estimate register pressure of the vectorized code. */ void ix86_vect_estimate_reg_pressure (); /* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for @@ -26468,6 +26469,8 @@ private: unsigned m_num_reduc[X86_REDUC_LAST]; /* Don't do unroll if m_prefer_unroll is false, default is true. */ bool m_prefer_unroll; + /* Scalar lanes in fold-left reductions. */ + unsigned int m_num_fold_left_reduc_lanes; }; ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar) @@ -26477,7 +26480,8 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar) m_num_avx256_vec_perm (), m_num_avx512_vec_perm (), m_num_reduc (), - m_prefer_unroll (true) + m_prefer_unroll (true), + m_num_fold_left_reduc_lanes (0) {} /* Implement targetm.vectorize.create_costs. */ @@ -27183,6 +27187,20 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs) loop_vec_info loop_vinfo = dyn_cast (m_vinfo); if (loop_vinfo && !m_costing_for_scalar) { + unsigned int vf = vect_vf_for_cost (loop_vinfo); + for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo)) + if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group + || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain) + && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst)) + == FOLD_LEFT_REDUCTION)) + m_num_fold_left_reduc_lanes + += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst)); + + if (m_num_fold_left_reduc_lanes && dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "in-order FP reduction lanes: %u\n", + m_num_fold_left_reduc_lanes); + /* We are currently not asking the vectorizer to compare costs between different vector mode sizes. When using predication that will end up always choosing the preferred mode size even @@ -27347,6 +27365,21 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs) vector_costs::finish_cost (scalar_costs); } +/* Return true if THIS has a shorter fold-left reduction chain than OTHER. */ + +bool +ix86_vector_costs::better_fold_left_reduc_than_p + (const vector_costs *other) const +{ + auto other_costs = static_cast (other); + if (m_num_fold_left_reduc_lanes >= other_costs->m_num_fold_left_reduc_lanes) + return false; + + /* Do not let emulated sub-SSE modes win on this alone. */ + loop_vec_info loop_vinfo = as_a (m_vinfo); + return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16); +} + /* Return true if THIS should be preferred over OTHER as main vector loop. */ bool @@ -27355,6 +27388,9 @@ ix86_vector_costs::better_main_loop_than_p (const vector_costs *other) const loop_vec_info this_loop_vinfo = as_a (this->vinfo ()); loop_vec_info other_loop_vinfo = as_a (other->vinfo ()); + if (better_fold_left_reduc_than_p (other)) + return true; + /* If the other loop is masked it does not need an epilog. Prefer that if the current loop cannot be vectorized fully with a vector epilogs with at most one scalar iteration left. */ @@ -27377,6 +27413,10 @@ ix86_vector_costs::better_epilogue_loop_than_p (const vector_costs *other, loop_vec_info main_loop) const { loop_vec_info this_loop_info = as_a (this->vinfo ()); + + if (better_fold_left_reduc_than_p (other)) + return true; + /* The x86 target allows for multiple vector epilogues, if THIS is the suggested epilog mode of OTHER then keep the latter unless THIS has a VF of one which means no further epilog needed. */ diff --git a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c index abb9f6681c2..370e7ed4d11 100644 --- a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c +++ b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c @@ -52,7 +52,6 @@ void test (const unsigned char * __restrict__ pi, pp_avg_rgb[3] = pp_avg_rgb_3; } -/* Even though there's an SLP opportunity in-order reductions should never use - masked epilogs. */ -/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */ -/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32 byte vectors" "vect" } } */ +/* In-order reductions should not use masked epilogs here. */ +/* { dg-final { scan-tree-dump "optimized: loop vectorized using 16 byte vectors" "vect" } } */ +/* { dg-final { scan-tree-dump-not "optimized: epilogue loop vectorized using masked" "vect" } } */ diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c new file mode 100644 index 00000000000..634647287d8 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c @@ -0,0 +1,19 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */ + +/* The two dependent updates form a fold-left reduction chain. */ + +float +foo (float *__restrict__ a, int n) +{ + float sum = 0; + for (int i = 0; i != n; i++) + { + sum += a[2 * i]; + sum += a[2 * i + 1]; + } + return sum; +} + +/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain" "vect" } } */ +/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */ diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c new file mode 100644 index 00000000000..6babd761d6f --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c @@ -0,0 +1,18 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */ + +/* The byte loads make V32QI available, but the fold-left reduction should + make the smaller SSE loop win. */ + +float +foo (char *a, char *b, int n) +{ + float sum = 0; + for (int i = 0; i != n; i++) + sum += a[i] * b[i]; + return sum; +} + +/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */ +/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect" } } */ +/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors" "vect" } } */ diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c index 8cd8740c6ec..9f0445390f4 100644 --- a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c +++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c @@ -17,5 +17,4 @@ foo (double *a, char *mask, int n) return sum; } -/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */ -/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32 byte vectors" "vect" } } */ +/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte vectors" "vect" } } */ diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c index 63c29895f9b..be56819fc54 100644 --- a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c +++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c @@ -17,5 +17,5 @@ foo (double *a, char *mask, int n) return sum; } -/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */ -/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using masked 64 byte vectors" "vect" } } */ +/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte vectors" "vect" } } */ +/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using masked 32 byte vectors" "vect" } } */