From: David Benjamin <davidben@google.com>
Date: Tue, 14 Jun 2022 03:29:07 +0000 (-0400)
Subject: Simplify BN_GENCB handling
X-Git-Tag: v9.19.4~21^2~1
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=723f5a0769d8ca9fabfeab81e6efd9922df81960;p=thirdparty%2Fbind9.git

Simplify BN_GENCB handling

When callback was NULL, bind9 would use BN_GENCB_set_old to set a NULL
callback because OpenSSL happened to allow a NULL "old" callback, but
not a NULL "new" callback. Instead, the way to turn off the callback is
to pass a NULL BN_GENCB itself.

Switch to doing that.
---

diff --git a/lib/dns/openssldh_link.c b/lib/dns/openssldh_link.c
index d5dbc2e8899..0a2e11f51fa 100644
--- a/lib/dns/openssldh_link.c
+++ b/lib/dns/openssldh_link.c
@@ -444,16 +444,14 @@ openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
 
 	if (generator != 0) {
 #if OPENSSL_VERSION_NUMBER < 0x30000000L
-		cb = BN_GENCB_new();
+		if (callback != NULL) {
+			cb = BN_GENCB_new();
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-		if (cb == NULL) {
-			DST_RET(dst__openssl_toresult(ISC_R_NOMEMORY));
-		}
+			if (cb == NULL) {
+				DST_RET(dst__openssl_toresult(ISC_R_NOMEMORY));
+			}
 #endif /* if OPENSSL_VERSION_NUMBER >= 0x10100000L && \
 	* !defined(LIBRESSL_VERSION_NUMBER) */
-		if (callback == NULL) {
-			BN_GENCB_set_old(cb, NULL, NULL);
-		} else {
 			u.fptr = callback;
 			BN_GENCB_set(cb, progress_cb, u.dptr);
 		}
diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index be1998e3819..26c11828056 100644
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -384,14 +384,14 @@ opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 #if !HAVE_BN_GENCB_NEW
 	BN_GENCB _cb;
 #endif /* !HAVE_BN_GENCB_NEW */
-	BN_GENCB *cb = BN_GENCB_new();
+	BN_GENCB *cb = NULL;
 #else
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_from_name(NULL, "RSA", NULL);
 	EVP_PKEY *pkey = NULL;
 #endif /* OPENSSL_VERSION_NUMBER < 0x30000000L */
 
 #if OPENSSL_VERSION_NUMBER < 0x30000000L
-	if (e == NULL || rsa == NULL || pkey == NULL || cb == NULL) {
+	if (e == NULL || rsa == NULL || pkey == NULL) {
 		DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 	}
 #else
@@ -442,9 +442,14 @@ opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 		DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 	}
 
-	if (callback == NULL) {
-		BN_GENCB_set_old(cb, NULL, NULL);
-	} else {
+	if (callback != NULL) {
+		cb = BN_GENCB_new();
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+		if (cb == NULL) {
+			DST_RET(dst__openssl_toresult(ISC_R_NOMEMORY));
+		}
+#endif /* if OPENSSL_VERSION_NUMBER >= 0x10100000L && \
+	* !defined(LIBRESSL_VERSION_NUMBER) */
 		u.fptr = callback;
 		BN_GENCB_set(cb, progress_cb, u.dptr);
 	}