factor: use 64-bit internal counters

* src/factor.c (factor_using_pollard_rho)
(factor_using_pollard_rho2, mp_factor_using_pollard_rho):
Use int_fast64_t for internal counters rather than int, as int
could overflow on some somewhat-practical examples.  Problem
discovered on a hypothetical platform where W_TYPE_SIZE is neither
32 nor 64, when factoring 0x7ffffffffffffeab7fffffffffff7369 ==
170141183460469225450570946617781744489, causing k to overflow in
mp_factor_using_pollard_rho.  Presumably a similar problem exists
in the previous stable coreutils 9.7, too, on 32-bit platforms
with somewhat-larger test cases, though I haven’t take the
somewhat-extensive CPU time to discover it.

diff --git a/src/factor.c b/src/factor.c
index fe3a639a14e0fc66073fdbd2004bcee8f22bf099..548583a3fb4f09d40de30c24e96fda84e1781df3 100644 (file)
--- a/src/factor.c
+++ b/src/factor.c
@@ -1205,8 +1205,8 @@ factor_using_pollard_rho (struct factors *factors, mp_limb_t n, mp_limb_t a)
 {
   mp_limb_t x, z, y, P, t, ni, g;
 
-  int k = 1;
-  int l = 1;
+  int_fast64_t k = 1;
+  int_fast64_t l = 1;
 
   redcify (P, 1, n);
   addmod (x, P, P, n);          /* i.e., redcify(2) */
@@ -1240,7 +1240,7 @@ factor_using_pollard_rho (struct factors *factors, mp_limb_t n, mp_limb_t a)
           z = x;
           k = l;
           l = 2 * l;
-          for (int i = 0; i < k; i++)
+          for (int_fast64_t i = 0; i < k; i++)
             {
               x = mulredc (x, x, n, ni);
               addmod (x, x, a, n);
@@ -1291,8 +1291,8 @@ factor_using_pollard_rho2 (struct factors *factors,
 {
   mp_limb_t x1, x0, z1, z0, y1, y0, P1, P0, t1, t0, g1, g0, r1m;
 
-  int k = 1;
-  int l = 1;
+  int_fast64_t k = 1;
+  int_fast64_t l = 1;
 
   redcify2 (P1, P0, 1, n1, n0);
   addmod2 (x1, x0, P1, P0, P1, P0, n1, n0); /* i.e., redcify(2) */
@@ -1328,7 +1328,7 @@ factor_using_pollard_rho2 (struct factors *factors,
           z1 = x1; z0 = x0;
           k = l;
           l = 2 * l;
-          for (int i = 0; i < k; i++)
+          for (int_fast64_t i = 0; i < k; i++)
             {
               x0 = mulredc2 (&r1m, x1, x0, x1, x0, n1, n0, ni);
               x1 = r1m;
@@ -1487,9 +1487,9 @@ mp_factor_using_pollard_rho (struct mp_factors *factors,
 
   mp_limb_t m0inv = binv_limb (-mp[0]);
 
-  for (int k = 1; ; k *= 2)
+  for (int_fast64_t k = 1; ; k *= 2)
     {
-      for (int i = k; 0 < i; i--)
+      for (int_fast64_t i = k; 0 < i; i--)
         {
           mp_mulredc (tp, xp, xp, mp, n, m0inv, scratch);
           mp_modadd_1 (xp, tp, a, mp, n);
@@ -1514,7 +1514,7 @@ mp_factor_using_pollard_rho (struct mp_factors *factors,
         }
 
       mpn_copyi (zp, xp, n);
-      for (int i = 2 * k; 0 < i; i--)
+      for (int_fast64_t i = 2 * k; 0 < i; i--)
         {
           mp_mulredc (tp, xp, xp, mp, n, m0inv, scratch);
           mp_modadd_1 (xp, tp, a, mp, n);