Fix multiple wrong use of BN_check_prime

Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25811)

diff --git a/apps/prime.c b/apps/prime.c
index a03909c411f06a04c2a953ac364e7e7471c67706..23b7e462fc20d2b0d03212fa034f4cb7aad8a1ce 100644 (file)
--- a/apps/prime.c
+++ b/apps/prime.c
@@ -145,10 +145,14 @@ opthelp:
             }
 
             BN_print(bio_out, bn);
+            r = BN_check_prime(bn, NULL, NULL);
+            if (r < 0) {
+                BIO_printf(bio_err, "Error checking prime\n");
+                goto end;
+            }
             BIO_printf(bio_out, " (%s) %s prime\n",
                        argv[0],
-                       BN_check_prime(bn, NULL, NULL)
-                           ? "is" : "is not");
+                       r == 1 ? "is" : "is not");
         }
     }
 

diff --git a/test/bntest.c b/test/bntest.c
index 2c7b91fad960a9bed1c06e614055c6e21c792ae4..3b3671a62259842a1b89177287d6fd6a20deaa23 100644 (file)
--- a/test/bntest.c
+++ b/test/bntest.c
@@ -2703,7 +2703,7 @@ static int test_not_prime(int i)
 
     for (trial = 0; trial <= 1; ++trial) {
         if (!TEST_true(BN_set_word(r, not_primes[i]))
-                || !TEST_false(BN_check_prime(r, ctx, NULL)))
+                || !TEST_int_eq(BN_check_prime(r, ctx, NULL), 0))
             goto err;
     }