[PR103437] Process multiplication overflow in priority calculation for allocno assign...

We process overflows in cost calculations but for huge functions
priority calculation can overflow as priority can be bigger the cost
used for it.  The patch fixes the problem.

gcc/ChangeLog:

	PR rtl-optimization/103437
	* ira-color.c (setup_allocno_priorities): Process multiplication
	overflow.

diff --git a/gcc/ira-color.c b/gcc/ira-color.c
index 3d01c60800c5dd666d9581c0adbcfff02872fb75..1f80cbea0e2fc4975efeb5343547be7ae81969ac 100644 (file)
--- a/gcc/ira-color.c
+++ b/gcc/ira-color.c
@@ -2796,7 +2796,7 @@ static int *allocno_priorities;
 static void
 setup_allocno_priorities (ira_allocno_t *consideration_allocnos, int n)
 {
-  int i, length, nrefs, priority, max_priority, mult;
+  int i, length, nrefs, priority, max_priority, mult, diff;
   ira_allocno_t a;
 
   max_priority = 0;
@@ -2807,11 +2807,14 @@ setup_allocno_priorities (ira_allocno_t *consideration_allocnos, int n)
       ira_assert (nrefs >= 0);
       mult = floor_log2 (ALLOCNO_NREFS (a)) + 1;
       ira_assert (mult >= 0);
-      allocno_priorities[ALLOCNO_NUM (a)]
-       = priority
-       = (mult
-          * (ALLOCNO_MEMORY_COST (a) - ALLOCNO_CLASS_COST (a))
-          * ira_reg_class_max_nregs[ALLOCNO_CLASS (a)][ALLOCNO_MODE (a)]);
+      mult *= ira_reg_class_max_nregs[ALLOCNO_CLASS (a)][ALLOCNO_MODE (a)];
+      diff = ALLOCNO_MEMORY_COST (a) - ALLOCNO_CLASS_COST (a);
+      /* Multiplication can overflow for very large functions.
+        Check the overflow and constrain the result if necessary: */
+      if (__builtin_smul_overflow (mult, diff, &priority)
+         || priority <= -INT_MAX)
+       priority = diff >= 0 ? INT_MAX : -INT_MAX;
+      allocno_priorities[ALLOCNO_NUM (a)] = priority;
       if (priority < 0)
        priority = -priority;
       if (max_priority < priority)