Remove OpenSSL memory tracking support from the ossl3.c module

OPENSSL_cleanup() in OpenSSL 4 doesn't free the memory, and that is
not compatible with BIND 9's memory leak detection code. Don't use
custom allocation/deallocation functions for OpenSSL's internal memory
management in the ossl3.c module.

See https://github.com/openssl/openssl/pull/29721

diff --git a/lib/isc/crypto/ossl3.c b/lib/isc/crypto/ossl3.c
index 1c2f3cab9da7a6ff89093ca3962104820acabc60..35344dc8b1b3547a1ebe9fec37979b02d0b928ba 100644 (file)
--- a/lib/isc/crypto/ossl3.c
+++ b/lib/isc/crypto/ossl3.c
@@ -44,8 +44,6 @@ struct isc_hmac_key {
 
 constexpr uint32_t hmac_key_magic = ISC_MAGIC('H', 'M', 'A', 'C');
 
-static isc_mem_t *isc__crypto_mctx = NULL;
-
 static OSSL_PROVIDER *base = NULL, *fips = NULL;
 
 static EVP_MAC *evp_hmac = NULL;
@@ -320,69 +318,6 @@ isc_hmac_final(isc_hmac_t *hmac, isc_buffer_t *out) {
        return ISC_R_SUCCESS;
 }
 
-#if ISC_MEM_TRACKLINES
-/*
- * We use the internal isc__mem API here, so we can pass the file and line
- * arguments passed from OpenSSL >= 1.1.0 to our memory functions for better
- * tracking of the OpenSSL allocations.  Without this, we would always just see
- * isc__crypto_{malloc,realloc,free} in the tracking output, but with this in
- * place we get to see the places in the OpenSSL code where the allocations
- * happen.
- */
-
-static void *
-isc__crypto_malloc_ex(size_t size, const char *file, int line) {
-       return isc__mem_allocate(isc__crypto_mctx, size, 0, __func__, file,
-                                (unsigned int)line);
-}
-
-static void *
-isc__crypto_realloc_ex(void *ptr, size_t size, const char *file, int line) {
-       return isc__mem_reallocate(isc__crypto_mctx, ptr, size, 0, __func__,
-                                  file, (unsigned int)line);
-}
-
-static void
-isc__crypto_free_ex(void *ptr, const char *file, int line) {
-       if (ptr == NULL) {
-               return;
-       }
-       if (isc__crypto_mctx != NULL) {
-               isc__mem_free(isc__crypto_mctx, ptr, 0, __func__, file,
-                             (unsigned int)line);
-       }
-}
-
-#else /* ISC_MEM_TRACKLINES */
-
-static void *
-isc__crypto_malloc_ex(size_t size, const char *file, int line) {
-       UNUSED(file);
-       UNUSED(line);
-       return isc_mem_allocate(isc__crypto_mctx, size);
-}
-
-static void *
-isc__crypto_realloc_ex(void *ptr, size_t size, const char *file, int line) {
-       UNUSED(file);
-       UNUSED(line);
-       return isc_mem_reallocate(isc__crypto_mctx, ptr, size);
-}
-
-static void
-isc__crypto_free_ex(void *ptr, const char *file, int line) {
-       UNUSED(file);
-       UNUSED(line);
-       if (ptr == NULL) {
-               return;
-       }
-       if (isc__crypto_mctx != NULL) {
-               isc__mem_free(isc__crypto_mctx, ptr, 0);
-       }
-}
-
-#endif /* ISC_MEM_TRACKLINES */
-
 bool
 isc_crypto_fips_mode(void) {
        return EVP_default_properties_is_fips_enabled(NULL) != 0;
@@ -424,9 +359,17 @@ isc_crypto_fips_enable(void) {
        return ISC_R_SUCCESS;
 }
 
+/*
+ * OPENSSL_cleanup() in OpenSSL 4 doesn't free the memory, which is not
+ * compatible with BIND 9's memory leak detection code, that is why the memory
+ * tracking has been disabled in this module, and this function is a no-op.
+ * This can be cleaned up once OpenSSL 1.1.x support is removed.
+ *
+ * See https://github.com/openssl/openssl/pull/29721
+ */
 void
 isc__crypto_setdestroycheck(bool check) {
-       isc_mem_setdestroycheck(isc__crypto_mctx, check);
+       UNUSED(check);
 }
 
 void
@@ -437,19 +380,6 @@ isc__crypto_initialize(void) {
         */
        uint64_t opts = OPENSSL_INIT_LOAD_CONFIG | OPENSSL_INIT_NO_ATEXIT;
 
-       isc_mem_create("OpenSSL", &isc__crypto_mctx);
-       isc_mem_setdebugging(isc__crypto_mctx, 0);
-       isc_mem_setdestroycheck(isc__crypto_mctx, false);
-
-       /*
-        * CRYPTO_set_mem_(_ex)_functions() returns 1 on success or 0 on
-        * failure, which means OpenSSL already allocated some memory.  There's
-        * nothing we can do about it.
-        */
-       (void)CRYPTO_set_mem_functions(isc__crypto_malloc_ex,
-                                      isc__crypto_realloc_ex,
-                                      isc__crypto_free_ex);
-
        RUNTIME_CHECK(OPENSSL_init_ssl(opts, NULL) == 1);
 
        register_algorithms();
@@ -486,6 +416,4 @@ isc__crypto_shutdown(void) {
        }
 
        OPENSSL_cleanup();
-
-       isc_mem_detach(&isc__crypto_mctx);
 }