Fix also BIO_printf formatting for INF and NAN

Avoid infinite loooooooops in %e and %g formatting for +/-INF and
make the invalid value at least visible by using '?' as signvalue.

Fixes #26973

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/27491)

(cherry picked from commit b56dd5bfec8e790cc2d5b1bdca6ecd350a3b7779)

diff --git a/crypto/bio/bio_print.c b/crypto/bio/bio_print.c
index 254c7ab0441a5648d57b9df4bfc3f72477c50429..19d2a613e9ac396ceb1c75749197459180ec3207 100644 (file)
--- a/crypto/bio/bio_print.c
+++ b/crypto/bio/bio_print.c
@@ -535,6 +535,10 @@ static LDOUBLE abs_val(LDOUBLE value)
     LDOUBLE result = value;
     if (value < 0)
         result = -value;
+    if (result > 0 && result / 2 == result) /* INF */
+        result = 0;
+    else if (result != result) /* NAN */
+        result = 0;
     return result;
 }
 
@@ -591,6 +595,8 @@ fmtfp(char **sbuffer,
     else if (flags & DP_F_SPACE)
         signvalue = ' ';
     ufvalue = abs_val(fvalue);
+    if (ufvalue == 0 && fvalue != 0) /* INF or NAN? */
+        signvalue = '?';
 
     /*
      * G_FORMAT sometimes prints like E_FORMAT and sometimes like F_FORMAT