params: fix range check when converting double to uint64_t.

Found in #15815

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15819)

diff --git a/crypto/params.c b/crypto/params.c
index a1db5dba2681e1b4748452889787a1a15e6474d1..9049041e3bbe70447a7a11e691e4b2d8ee955849 100644 (file)
--- a/crypto/params.c
+++ b/crypto/params.c
@@ -1029,7 +1029,7 @@ int OSSL_PARAM_set_double(OSSL_PARAM *p, double val)
                      * 15 bits of UINT64_MAX to avoid using imprecise floating
                      * point values.
                      */
-                    && (double)(UINT64_MAX - 65535) + 65536.0) {
+                    && val < (double)(UINT64_MAX - 65535) + 65536.0) {
                 p->return_size = sizeof(uint64_t);
                 *(uint64_t *)p->data = (uint64_t)val;
                 return 1;