4129. [port] Address API changes in OpenSSL 1.1.0. [RT #39532]

diff --git a/CHANGES b/CHANGES
index fa215da4c47468c2109c232c971c2872618f7063..e7f2cb6698cffd4735e00014c50ed62d57733478 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,5 @@
+4129.  [port]          Address API changes in OpenSSL 1.1.0. [RT #39532]
+
 4128.  [bug]           Address issues raised by Coverity 7.6. [RT #39537]
 
 4127.  [protocol]      CDS and CDNSKEY need to be signed by the key signing

diff --git a/lib/dns/dst_openssl.h b/lib/dns/dst_openssl.h
index 99a43ef948aca105624783a2508a0ca13690f0ff..73ec30e772552ec94214560e8d9788c4d74d844f 100644 (file)
--- a/lib/dns/dst_openssl.h
+++ b/lib/dns/dst_openssl.h
@@ -29,12 +29,27 @@
 #include <openssl/evp.h>
 #include <openssl/conf.h>
 #include <openssl/crypto.h>
+#include <openssl/bn.h>
 
 #if !defined(OPENSSL_NO_ENGINE) && defined(CRYPTO_LOCK_ENGINE) && \
     (OPENSSL_VERSION_NUMBER >= 0x0090707f)
 #define USE_ENGINE 1
 #endif
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+/*
+ * These are new in OpenSSL 1.1.0.  BN_GENCB _cb needs to be declared in
+ * the function like this before the BN_GENCB_new call:
+ *
+ * #if OPENSSL_VERSION_NUMBER < 0x10100000L
+ *              _cb;
+ * #endif
+ */
+#define BN_GENCB_free(x) (x = NULL);
+#define BN_GENCB_new() (&_cb)
+#define BN_GENCB_get_arg(x) ((x)->arg)
+#endif
+
 ISC_LANG_BEGINDECLS
 
 isc_result_t

diff --git a/lib/dns/openssl_link.c b/lib/dns/openssl_link.c
index a0a2724f41c1437db048b37c8d6a18575499cc33..1c9d102ebe08dc1cc88bdfb913dd8b29a16602e8 100644 (file)
--- a/lib/dns/openssl_link.c
+++ b/lib/dns/openssl_link.c
@@ -88,6 +88,7 @@ entropy_getpseudo(unsigned char *buf, int num) {
        return (result == ISC_R_SUCCESS ? 1 : -1);
 }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 static void
 entropy_add(const void *buf, int num, double entropy) {
        /*
@@ -97,6 +98,18 @@ entropy_add(const void *buf, int num, double entropy) {
        UNUSED(num);
        UNUSED(entropy);
 }
+#else
+static int
+entropy_add(const void *buf, int num, double entropy) {
+       /*
+        * Do nothing.  The only call to this provides no useful data anyway.
+        */
+       UNUSED(buf);
+       UNUSED(num);
+       UNUSED(entropy);
+       return (1);
+}
+#endif
 
 static void
 lock_callback(int mode, int type, const char *file, int line) {
@@ -108,10 +121,12 @@ lock_callback(int mode, int type, const char *file, int line) {
                UNLOCK(&locks[type]);
 }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 static unsigned long
 id_callback(void) {
        return ((unsigned long)isc_thread_self());
 }
+#endif
 
 static void *
 mem_alloc(size_t size) {
@@ -172,7 +187,9 @@ dst__openssl_init(const char *engine) {
        if (result != ISC_R_SUCCESS)
                goto cleanup_mutexalloc;
        CRYPTO_set_locking_callback(lock_callback);
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        CRYPTO_set_id_callback(id_callback);
+#endif
 
        ERR_load_crypto_strings();
 
@@ -270,7 +287,9 @@ dst__openssl_destroy(void) {
        CRYPTO_cleanup_all_ex_data();
 #endif
        ERR_clear_error();
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        ERR_remove_state(0);
+#endif
        ERR_free_strings();
 
 #ifdef  DNS_CRYPTO_LEAKS

diff --git a/lib/dns/openssldh_link.c b/lib/dns/openssldh_link.c
index ec74d8a581f0ae1eed3562fac22a9f3a7e71fab6..dbda260d7271b9e9df6d19c2b560c78c4bb8fa23 100644 (file)
--- a/lib/dns/openssldh_link.c
+++ b/lib/dns/openssldh_link.c
@@ -71,7 +71,7 @@
 
 static isc_result_t openssldh_todns(const dst_key_t *key, isc_buffer_t *data);
 
-static BIGNUM bn2, bn768, bn1024, bn1536;
+static BIGNUM *bn2, *bn768, *bn1024, *bn1536;
 
 static isc_result_t
 openssldh_computesecret(const dst_key_t *pub, const dst_key_t *priv,
@@ -161,7 +161,7 @@ progress_cb(int p, int n, BN_GENCB *cb)
 
        UNUSED(n);
 
-       u.dptr = cb->arg;
+       u.dptr = BN_GENCB_get_arg(cb);
        if (u.fptr != NULL)
                u.fptr(p);
        return (1);
@@ -172,7 +172,10 @@ static isc_result_t
 openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
        DH *dh = NULL;
 #if OPENSSL_VERSION_NUMBER > 0x00908000L
-       BN_GENCB cb;
+       BN_GENCB *cb;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+       BN_GENCB _cb;
+#endif
        union {
                void *dptr;
                void (*fptr)(int);
@@ -191,12 +194,12 @@ openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
                        if (dh == NULL)
                                return (dst__openssl_toresult(ISC_R_NOMEMORY));
                        if (key->key_size == 768)
-                               dh->p = &bn768;
+                               dh->p = bn768;
                        else if (key->key_size == 1024)
-                               dh->p = &bn1024;
+                               dh->p = bn1024;
                        else
-                               dh->p = &bn1536;
-                       dh->g = &bn2;
+                               dh->p = bn1536;
+                       dh->g = bn2;
                } else
                        generator = 2;
        }
@@ -206,21 +209,28 @@ openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
                dh = DH_new();
                if (dh == NULL)
                        return (dst__openssl_toresult(ISC_R_NOMEMORY));
+               cb = BN_GENCB_new();
+               if (cb == NULL) {
+                       DH_free(dh);
+                       return (dst__openssl_toresult(ISC_R_NOMEMORY));
+               }
 
                if (callback == NULL) {
-                       BN_GENCB_set_old(&cb, NULL, NULL);
+                       BN_GENCB_set_old(cb, NULL, NULL);
                } else {
                        u.fptr = callback;
-                       BN_GENCB_set(&cb, &progress_cb, u.dptr);
+                       BN_GENCB_set(cb, &progress_cb, u.dptr);
                }
 
                if (!DH_generate_parameters_ex(dh, key->key_size, generator,
-                                              &cb)) {
+                                              cb)) {
                        DH_free(dh);
+                       BN_GENCB_free(cb);
                        return (dst__openssl_toresult2(
                                        "DH_generate_parameters_ex",
                                        DST_R_OPENSSLFAILURE));
                }
+               BN_GENCB_free(cb);
 #else
                dh = DH_generate_parameters(key->key_size, generator,
                                            NULL, NULL);
@@ -256,9 +266,9 @@ openssldh_destroy(dst_key_t *key) {
        if (dh == NULL)
                return;
 
-       if (dh->p == &bn768 || dh->p == &bn1024 || dh->p == &bn1536)
+       if (dh->p == bn768 || dh->p == bn1024 || dh->p == bn1536)
                dh->p = NULL;
-       if (dh->g == &bn2)
+       if (dh->g == bn2)
                dh->g = NULL;
        DH_free(dh);
        key->keydata.dh = NULL;
@@ -294,8 +304,8 @@ openssldh_todns(const dst_key_t *key, isc_buffer_t *data) {
 
        isc_buffer_availableregion(data, &r);
 
-       if (dh->g == &bn2 &&
-           (dh->p == &bn768 || dh->p == &bn1024 || dh->p == &bn1536)) {
+       if (dh->g == bn2 &&
+           (dh->p == bn768 || dh->p == bn1024 || dh->p == bn1536)) {
                plen = 1;
                glen = 0;
        }
@@ -310,9 +320,9 @@ openssldh_todns(const dst_key_t *key, isc_buffer_t *data) {
 
        uint16_toregion(plen, &r);
        if (plen == 1) {
-               if (dh->p == &bn768)
+               if (dh->p == bn768)
                        *r.base = 1;
-               else if (dh->p == &bn1024)
+               else if (dh->p == bn1024)
                        *r.base = 2;
                else
                        *r.base = 3;
@@ -375,13 +385,13 @@ openssldh_fromdns(dst_key_t *key, isc_buffer_t *data) {
                        special = uint16_fromregion(&r);
                switch (special) {
                        case 1:
-                               dh->p = &bn768;
+                               dh->p = bn768;
                                break;
                        case 2:
-                               dh->p = &bn1024;
+                               dh->p = bn1024;
                                break;
                        case 3:
-                               dh->p = &bn1536;
+                               dh->p = bn1536;
                                break;
                        default:
                                DH_free(dh);
@@ -409,12 +419,12 @@ openssldh_fromdns(dst_key_t *key, isc_buffer_t *data) {
        }
        if (special != 0) {
                if (glen == 0)
-                       dh->g = &bn2;
+                       dh->g = bn2;
                else {
                        dh->g = BN_bin2bn(r.base, glen, NULL);
-                       if (BN_cmp(dh->g, &bn2) == 0) {
+                       if (BN_cmp(dh->g, bn2) == 0) {
                                BN_free(dh->g);
-                               dh->g = &bn2;
+                               dh->g = bn2;
                        }
                        else {
                                DH_free(dh);
@@ -569,25 +579,25 @@ openssldh_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
        if ((key->key_size == 768 ||
             key->key_size == 1024 ||
             key->key_size == 1536) &&
-           BN_cmp(dh->g, &bn2) == 0)
+           BN_cmp(dh->g, bn2) == 0)
        {
-               if (key->key_size == 768 && BN_cmp(dh->p, &bn768) == 0) {
+               if (key->key_size == 768 && BN_cmp(dh->p, bn768) == 0) {
                        BN_free(dh->p);
                        BN_free(dh->g);
-                       dh->p = &bn768;
-                       dh->g = &bn2;
+                       dh->p = bn768;
+                       dh->g = bn2;
                } else if (key->key_size == 1024 &&
-                          BN_cmp(dh->p, &bn1024) == 0) {
+                          BN_cmp(dh->p, bn1024) == 0) {
                        BN_free(dh->p);
                        BN_free(dh->g);
-                       dh->p = &bn1024;
-                       dh->g = &bn2;
+                       dh->p = bn1024;
+                       dh->g = bn2;
                } else if (key->key_size == 1536 &&
-                          BN_cmp(dh->p, &bn1536) == 0) {
+                          BN_cmp(dh->p, bn1536) == 0) {
                        BN_free(dh->p);
                        BN_free(dh->g);
-                       dh->p = &bn1536;
-                       dh->g = &bn2;
+                       dh->p = bn1536;
+                       dh->g = bn2;
                }
        }
 
@@ -628,10 +638,10 @@ BN_fromhex(BIGNUM *b, const char *str) {
 
 static void
 openssldh_cleanup(void) {
-       BN_free(&bn2);
-       BN_free(&bn768);
-       BN_free(&bn1024);
-       BN_free(&bn1536);
+       BN_free(bn2);
+       BN_free(bn768);
+       BN_free(bn1024);
+       BN_free(bn1536);
 }
 
 static dst_func_t openssldh_functions = {
@@ -662,17 +672,27 @@ isc_result_t
 dst__openssldh_init(dst_func_t **funcp) {
        REQUIRE(funcp != NULL);
        if (*funcp == NULL) {
-               BN_init(&bn2);
-               BN_init(&bn768);
-               BN_init(&bn1024);
-               BN_init(&bn1536);
-               BN_set_word(&bn2, 2);
-               BN_fromhex(&bn768, PRIME768);
-               BN_fromhex(&bn1024, PRIME1024);
-               BN_fromhex(&bn1536, PRIME1536);
+               bn2 = BN_new();
+               bn768 = BN_new();
+               bn1024 = BN_new();
+               bn1536 = BN_new();
+               if (bn2 == NULL || bn768 == NULL ||
+                   bn1024 == NULL || bn1536 == NULL)
+                       goto cleanup;
+               BN_set_word(bn2, 2);
+               BN_fromhex(bn768, PRIME768);
+               BN_fromhex(bn1024, PRIME1024);
+               BN_fromhex(bn1536, PRIME1536);
                *funcp = &openssldh_functions;
        }
        return (ISC_R_SUCCESS);
+
+ cleanup:
+       if (bn2 != NULL) BN_free(bn2);
+       if (bn768 != NULL) BN_free(bn768);
+       if (bn1024 != NULL) BN_free(bn1024);
+       if (bn1536 != NULL) BN_free(bn1536);
+       return (ISC_R_NOMEMORY);
 }
 
 #else /* OPENSSL */

diff --git a/lib/dns/openssldsa_link.c b/lib/dns/openssldsa_link.c
index 67b6bbdae3637670867a9538eb66876430ecd2cd..227f08fb03bd2ad44d20e8ecb33f1251dc0b5f14 100644 (file)
--- a/lib/dns/openssldsa_link.c
+++ b/lib/dns/openssldsa_link.c
@@ -339,7 +339,7 @@ progress_cb(int p, int n, BN_GENCB *cb)
 
        UNUSED(n);
 
-       u.dptr = cb->arg;
+       u.dptr = BN_GENCB_get_arg(cb);
        if (u.fptr != NULL)
                u.fptr(p);
        return (1);
@@ -352,7 +352,10 @@ openssldsa_generate(dst_key_t *key, int unused, void (*callback)(int)) {
        unsigned char rand_array[ISC_SHA1_DIGESTLENGTH];
        isc_result_t result;
 #if OPENSSL_VERSION_NUMBER > 0x00908000L
-       BN_GENCB cb;
+       BN_GENCB *cb;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+       BN_GENCB _cb;
+#endif
        union {
                void *dptr;
                void (*fptr)(int);
@@ -373,22 +376,30 @@ openssldsa_generate(dst_key_t *key, int unused, void (*callback)(int)) {
        dsa = DSA_new();
        if (dsa == NULL)
                return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+       cb = BN_GENCB_new();
+       if (cb == NULL) {
+               DSA_free(dsa);
+               return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+       }
+
 
        if (callback == NULL) {
-               BN_GENCB_set_old(&cb, NULL, NULL);
+               BN_GENCB_set_old(cb, NULL, NULL);
        } else {
                u.fptr = callback;
-               BN_GENCB_set(&cb, &progress_cb, u.dptr);
+               BN_GENCB_set(cb, &progress_cb, u.dptr);
        }
 
        if (!DSA_generate_parameters_ex(dsa, key->key_size, rand_array,
                                        ISC_SHA1_DIGESTLENGTH,  NULL, NULL,
-                                       &cb))
+                                       cb))
        {
                DSA_free(dsa);
+               BN_GENCB_free(cb);
                return (dst__openssl_toresult2("DSA_generate_parameters_ex",
                                               DST_R_OPENSSLFAILURE));
        }
+       BN_GENCB_free(cb);
 #else
        dsa = DSA_generate_parameters(key->key_size, rand_array,
                                      ISC_SHA1_DIGESTLENGTH, NULL, NULL,

diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index 43d5df9e5e20c22758e1284fb7aa93dcc0251f69..886a5a1975e9c31caf154c7456f163c0c88440c7 100644 (file)
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -752,7 +752,7 @@ progress_cb(int p, int n, BN_GENCB *cb) {
 
        UNUSED(n);
 
-       u.dptr = cb->arg;
+       u.dptr = BN_GENCB_get_arg(cb);
        if (u.fptr != NULL)
                u.fptr(p);
        return (1);
@@ -763,18 +763,21 @@ static isc_result_t
 opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 #if OPENSSL_VERSION_NUMBER > 0x00908000L
        isc_result_t ret = DST_R_OPENSSLFAILURE;
-       BN_GENCB cb;
        union {
                void *dptr;
                void (*fptr)(int);
        } u;
        RSA *rsa = RSA_new();
        BIGNUM *e = BN_new();
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+       BN_GENCB _cb;
+#endif
+       BN_GENCB *cb = BN_GENCB_new();
 #if USE_EVP
        EVP_PKEY *pkey = EVP_PKEY_new();
 #endif
 
-       if (rsa == NULL || e == NULL)
+       if (rsa == NULL || e == NULL || cb == NULL)
                goto err;
 #if USE_EVP
        if (pkey == NULL)
@@ -794,14 +797,15 @@ opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
        }
 
        if (callback == NULL) {
-               BN_GENCB_set_old(&cb, NULL, NULL);
+               BN_GENCB_set_old(cb, NULL, NULL);
        } else {
                u.fptr = callback;
-               BN_GENCB_set(&cb, &progress_cb, u.dptr);
+               BN_GENCB_set(cb, &progress_cb, u.dptr);
        }
 
-       if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
+       if (RSA_generate_key_ex(rsa, key->key_size, e, cb)) {
                BN_free(e);
+               BN_GENCB_free(cb);
                SET_FLAGS(rsa);
 #if USE_EVP
                key->keydata.pkey = pkey;
@@ -812,6 +816,7 @@ opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 #endif
                return (ISC_R_SUCCESS);
        }
+       BN_GENCB_free(cb);
        ret = dst__openssl_toresult2("RSA_generate_key_ex",
                                     DST_R_OPENSSLFAILURE);
 
@@ -824,6 +829,8 @@ err:
                BN_free(e);
        if (rsa != NULL)
                RSA_free(rsa);
+       if (cb != NULL)
+               BN_GENCB_free(cb);
        return (dst__openssl_toresult(ret));
 #else
        RSA *rsa;

diff --git a/lib/dns/tests/gost_test.c b/lib/dns/tests/gost_test.c
index 7fca0a6d366f2f8b81260c2a04ab5ae32e3aaf02..4b65fda995022305b414fcb601585f1a22acce10 100644 (file)
--- a/lib/dns/tests/gost_test.c
+++ b/lib/dns/tests/gost_test.c
@@ -37,6 +37,7 @@
 #include <openssl/objects.h>
 #include <openssl/rsa.h>
 #include <openssl/engine.h>
+#include <openssl/bn.h>
 #endif
 
 #ifdef HAVE_PKCS11_GOST