openssl: Prevent OpenSSL from using posix_memalign() if LD is enabled

The leak detective doesn't wrap this function and calling the original
causes unknown memory frees and even segmentation faults.  This is now
triggered with OpenSSL 4 as the implementation of ECP256 uses
OPENSSL_aligned_alloc_array().

Setting a custom memory functions forces OpenSSL to implement aligned
allocations internally, using the registered allocation function.

diff --git a/src/libstrongswan/plugins/openssl/openssl_plugin.c b/src/libstrongswan/plugins/openssl/openssl_plugin.c
index ef7fe8908f3b723eb8211433864952646fb359a6..a12e2ba8fc986fec9ad7266f443c543eed92edb9 100644 (file)
--- a/src/libstrongswan/plugins/openssl/openssl_plugin.c
+++ b/src/libstrongswan/plugins/openssl/openssl_plugin.c
@@ -777,6 +777,13 @@ PLUGIN_DEFINE(openssl)
        private_openssl_plugin_t *this;
        int fips_mode;
 
+       /* prevent OpenSSL from using posix_memalign() if leak detective is enabled,
+        * which doesn't wrap it */
+       if (lib->leak_detective)
+       {
+               CRYPTO_set_mem_functions((void*)malloc, (void*)realloc, (void*)free);
+       }
+
        fips_mode = lib->settings->get_int(lib->settings,
                                                        "%s.plugins.openssl.fips_mode", FIPS_MODE, lib->ns);
 #ifdef OPENSSL_FIPS