factor: retry properly if Pollard rho gives a trivial factorization

* src/factor.c (factor_using_pollard_rho): Handle trivial factor g = n.
(factor_using_pollard_rho2): Handle trivial factor g1 = n1, g0 = n0.
* tests/misc/factor.pl: Add a test case.
Fixes http://bugs.gnu.org/25135

diff --git a/src/factor.c b/src/factor.c
index 115a635f9b9e46b21716c15201aef38f43f951f3..54893ca826753769f3926c238b681ad0afc37c78 100644 (file)
--- a/src/factor.c
+++ b/src/factor.c
@@ -1522,6 +1522,13 @@ factor_using_pollard_rho (uintmax_t n, unsigned long int a,
         }
       while (g == 1);
 
+      if (n == g)
+        {
+          /* Found n itself as factor.  Restart with different params.  */
+          factor_using_pollard_rho (n, a + 1, factors);
+          return;
+        }
+
       n = n / g;
 
       if (!prime_p (g))
@@ -1607,7 +1614,7 @@ factor_using_pollard_rho2 (uintmax_t n1, uintmax_t n0, unsigned long int a,
 
       if (g1 == 0)
         {
-          /* The found factor is one word. */
+          /* The found factor is one word, and > 1. */
           divexact_21 (n1, n0, n1, n0, g0);     /* n = n / g */
 
           if (!prime_p (g0))
@@ -1621,6 +1628,13 @@ factor_using_pollard_rho2 (uintmax_t n1, uintmax_t n0, unsigned long int a,
              to trigger.  Please be careful before you change this code!  */
           uintmax_t ginv;
 
+          if (n1 == g1 && n0 == g0)
+            {
+              /* Found n itself as factor.  Restart with different params.  */
+              factor_using_pollard_rho2 (n1, n0, a + 1, factors);
+              return;
+            }
+
           binv (ginv, g0);      /* Compute n = n / g.  Since the result will */
           n0 = ginv * n0;       /* fit one word, we can compute the quotient */
           n1 = 0;               /* modulo B, ignoring the high divisor word. */

diff --git a/tests/misc/factor.pl b/tests/misc/factor.pl
index e37df9d618f7a5149999d084793b23c4938a3e4e..ec5413745c4baad9437ebdafa0a4530a34dfb8f2 100755 (executable)
--- a/tests/misc/factor.pl
+++ b/tests/misc/factor.pl
@@ -78,6 +78,8 @@ my @Tests =
       {OUT => '3401347 3861211 12099721'}],
      ['bug-2016-b', '222087527029934481871',
       {OUT => '15601 26449 111427 4830277'}],
+     ['bug-2016-c', '12847291069740315094892340035',
+      {OUT => '5 4073 18899 522591721 63874247821'}],
     );
 
 # If we have GMP support, append tests to exercise it.