Fix error return values from BIO_ctrl_(w)pending()

Reviewed-by: Hugo Landau <hlandau@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19240)

diff --git a/crypto/bio/bio_lib.c b/crypto/bio/bio_lib.c
index f6d40b2d1c633da8bc57f88e95c318648f046a85..8a8c1f34f72deb54a2776d5abc487b85bb4d7d0a 100644 (file)
--- a/crypto/bio/bio_lib.c
+++ b/crypto/bio/bio_lib.c
@@ -714,12 +714,20 @@ long BIO_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp)
  */
 size_t BIO_ctrl_pending(BIO *bio)
 {
-    return BIO_ctrl(bio, BIO_CTRL_PENDING, 0, NULL);
+    long ret = BIO_ctrl(bio, BIO_CTRL_PENDING, 0, NULL);
+
+    if (ret < 0)
+        ret = 0;
+    return (size_t)ret;
 }
 
 size_t BIO_ctrl_wpending(BIO *bio)
 {
-    return BIO_ctrl(bio, BIO_CTRL_WPENDING, 0, NULL);
+    long ret = BIO_ctrl(bio, BIO_CTRL_WPENDING, 0, NULL);
+
+    if (ret < 0)
+        ret = 0;
+    return (size_t)ret;
 }
 
 /* put the 'bio' on the end of b's list of operators */

diff --git a/doc/man3/BIO_ctrl.pod b/doc/man3/BIO_ctrl.pod
index eb312bcd0cdb7bb735465e84c9dad7f973918162..f7e9217b1699244b2cebd1fa85444c0cac3422bb 100644 (file)
--- a/doc/man3/BIO_ctrl.pod
+++ b/doc/man3/BIO_ctrl.pod
@@ -112,7 +112,9 @@ BIO_get_close() returns the close flag value: BIO_CLOSE or BIO_NOCLOSE. It also
 returns other negative values if an error occurs.
 
 BIO_pending(), BIO_ctrl_pending(), BIO_wpending() and BIO_ctrl_wpending()
-return the amount of pending data.
+return the amount of pending data. BIO_pending() and BIO_wpending() return
+negative value or 0 on error. BIO_ctrl_pending() and BIO_ctrl_wpending() return
+0 on error.
 
 BIO_get_ktls_send() returns 1 if the BIO is using the Kernel TLS data-path for
 sending. Otherwise, it returns zero.
@@ -164,6 +166,9 @@ particular a return value of 0 can be returned if an operation is not
 supported, if an error occurred, if EOF has not been reached and in
 the case of BIO_seek() on a file BIO for a successful operation.
 
+In older versions of OpenSSL the BIO_ctrl_pending() and
+BIO_ctrl_wpending() could return values greater than INT_MAX on error.
+
 =head1 HISTORY
 
 The BIO_get_ktls_send() and BIO_get_ktls_recv() macros were added in