crypto/mem.c: perform the fail check right after counting calls

Otherwise the counting done by shouldfail() does not account for calls
that are diverted to non-standard implementation and zero-sized
allocations, making it diverge from the sum of malloc_count
and realloc_count.

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.foundation>
Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
MergeDate: Thu Apr 30 06:59:07 2026
(Merged from https://github.com/openssl/openssl/pull/30991)

diff --git a/crypto/mem.c b/crypto/mem.c
index 10252e4ae7e9cfc2f548d8989dc2b0e713954515..476d6b25293e985c43959b8d1031df90e2b0ff68 100644 (file)
--- a/crypto/mem.c
+++ b/crypto/mem.c
@@ -191,6 +191,7 @@ void *CRYPTO_malloc(size_t num, const char *file, int line)
     void *ptr;
 
     INCREMENT(malloc_count);
+    FAILTEST();
     if (malloc_impl != CRYPTO_malloc) {
         ptr = malloc_impl(num, file, line);
         if (ptr != NULL || num == 0)
@@ -201,7 +202,6 @@ void *CRYPTO_malloc(size_t num, const char *file, int line)
     if (ossl_unlikely(num == 0))
         return NULL;
 
-    FAILTEST();
     if (allow_customize) {
         /*
          * Disallow customization after the first allocation. We only set this
@@ -266,6 +266,7 @@ void *CRYPTO_realloc(void *str, size_t num, const char *file, int line)
     void *ret;
 
     INCREMENT(realloc_count);
+    FAILTEST();
     if (realloc_impl != CRYPTO_realloc) {
         ret = realloc_impl(str, num, file, line);
 
@@ -283,7 +284,6 @@ void *CRYPTO_realloc(void *str, size_t num, const char *file, int line)
         return NULL;
     }
 
-    FAILTEST();
     ret = realloc(str, num);
 
 err:

diff --git a/doc/man3/OPENSSL_malloc.pod b/doc/man3/OPENSSL_malloc.pod
index 0b7343bb862c9a5fa7d072170924bbb477ccdbc4..1907469fcd111df238b2075b1a344c65d6b94836 100644 (file)
--- a/doc/man3/OPENSSL_malloc.pod
+++ b/doc/man3/OPENSSL_malloc.pod
@@ -321,6 +321,10 @@ the caller may need to fall back to a non-aligned memory allocation
 Before OpenSSL 4.0, the call to OPENSSL_aligned_alloc() did not have
 an explicit upper limit on the value of I<alignment>.
 
+Before OpenSSL 4.1, allocations done by custom memory functions
+and zero-sized allocations did not progress allocation counter
+used against B<OPENSSL_MALLOC_FAILURES> specification.
+
 =head1 COPYRIGHT
 
 Copyright 2016-2025 The OpenSSL Project Authors. All Rights Reserved.