From ae13af26060eb686418ea9c9d455cd665049402d Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Sun, 23 Jun 2024 14:37:53 +0200 Subject: [PATCH] tree-optimization/115599 - reassoc qsort comparator issue The compare_repeat_factors comparator fails qsort checking eventually because it uses rf2->rank - rf1->rank to compare unsigned numbers which causes issues for ranks that interpret negative as signed. Fixed by re-writing the obvious way. I've also fixed the count comparison which suffers from truncation as count is 64bit signed while the comparator result is 32bit int (that's a lot less likely to hit in practice though). The testcase from the PR is too large to include. PR tree-optimization/115599 * tree-ssa-reassoc.cc (compare_repeat_factors): Use explicit compares to avoid truncations. --- gcc/tree-ssa-reassoc.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gcc/tree-ssa-reassoc.cc b/gcc/tree-ssa-reassoc.cc index 4d9f5216d4c..d74352268b5 100644 --- a/gcc/tree-ssa-reassoc.cc +++ b/gcc/tree-ssa-reassoc.cc @@ -6414,10 +6414,17 @@ compare_repeat_factors (const void *x1, const void *x2) const repeat_factor *rf1 = (const repeat_factor *) x1; const repeat_factor *rf2 = (const repeat_factor *) x2; - if (rf1->count != rf2->count) - return rf1->count - rf2->count; + if (rf1->count < rf2->count) + return -1; + else if (rf1->count > rf2->count) + return 1; + + if (rf1->rank < rf2->rank) + return 1; + else if (rf1->rank > rf2->rank) + return -1; - return rf2->rank - rf1->rank; + return 0; } /* Look for repeated operands in OPS in the multiply tree rooted at -- 2.47.2