Fix failure checking on thread_local storage assignment in rand_lib

The new malloc failure test caught an asan error in this code:
Direct leak of 40 byte(s) in 1 object(s) allocated from:
2025-08-07T03:22:20.3655117Z     #0 0x7fb88d8fd9c7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
2025-08-07T03:22:20.3655796Z     #1 0x5584f0e4670a in CRYPTO_malloc crypto/mem.c:211
2025-08-07T03:22:20.3656291Z     #2 0x5584f0e4679d in CRYPTO_zalloc crypto/mem.c:231
2025-08-07T03:22:20.3657040Z     #3 0x5584f11c4c10 in EVP_RAND_CTX_new crypto/evp/evp_rand.c:353
2025-08-07T03:22:20.3657656Z     #4 0x5584f0e93b27 in rand_new_drbg crypto/rand/rand_lib.c:666
2025-08-07T03:22:20.3658289Z     #5 0x5584f0e949d0 in rand_get0_public crypto/rand/rand_lib.c:843
2025-08-07T03:22:20.3658914Z     #6 0x5584f0e9305b in RAND_bytes_ex crypto/rand/rand_lib.c:490
2025-08-07T03:22:20.3659486Z     #7 0x5584f0b2405f in SSL_CTX_new_ex ssl/ssl_lib.c:4191
2025-08-07T03:22:20.3660183Z     #8 0x5584f0ae313c in create_ssl_ctx_pair test/helpers/ssltestlib.c:958
2025-08-07T03:22:20.3660871Z     #9 0x5584f0adeaf6 in do_handshake test/handshake-memfail.c:56
2025-08-07T03:22:20.3661539Z     #10 0x5584f0adee50 in test_alloc_failures test/handshake-memfail.c:125
2025-08-07T03:22:20.3662161Z     #11 0x5584f0cd9da8 in run_tests test/testutil/driver.c:342
2025-08-07T03:22:20.3662664Z     #12 0x5584f0cda9e5 in main test/testutil/main.c:31
2025-08-07T03:22:20.3663450Z     #13 0x7fb88d42a1c9  (/lib/x86_64-linux-gnu/libc.so.6+0x2a1c9) (BuildId: 282c2c16e7b6600b0b22ea0c99010d2795752b5f)
2025-08-07T03:22:20.3664630Z     #14 0x7fb88d42a28a in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2a28a) (BuildId: 282c2c16e7b6600b0b22ea0c99010d2795752b5f)
2025-08-07T03:22:20.3666608Z     #15 0x5584f0ade864 in _start (/home/runner/work/openssl/openssl/test/handshake-memfail+0x22a864) (BuildId: 19659a44d8bed2c082918d25425f77e3a98df534)

It occurs because when rand_get0_public/rand_get0_private sets an
EVP_RAND_CTX object in its thread local storage, it neglects to check
the return code of the operation, which may fail when the associated
sparse array is expanded.

fix it by checking the return code and failing the get0_[public|private]
operation so the failure is graceful.

Fixes openssl/project#1315

Reviewed-by: Paul Yang <paulyang.inf@gmail.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Paul Dale <ppzgs1@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/28195)

diff --git a/crypto/rand/rand_lib.c b/crypto/rand/rand_lib.c
index fedb6ed92ae3b85bc7efd1573ac0debe0c235894..c0f5c9effd162860d220365a0888836008668c5e 100644 (file)
--- a/crypto/rand/rand_lib.c
+++ b/crypto/rand/rand_lib.c
@@ -842,7 +842,10 @@ static EVP_RAND_CTX *rand_get0_public(OSSL_LIB_CTX *ctx, RAND_GLOBAL *dgbl)
             return NULL;
         rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL,
                              SECONDARY_RESEED_TIME_INTERVAL);
-        CRYPTO_THREAD_set_local_ex(CRYPTO_THREAD_LOCAL_DRBG_PUB_KEY, ctx, rand);
+        if (!CRYPTO_THREAD_set_local_ex(CRYPTO_THREAD_LOCAL_DRBG_PUB_KEY, ctx, rand)) {
+            EVP_RAND_CTX_free(rand);
+            rand = NULL;
+        }
     }
     return rand;
 }
@@ -882,7 +885,10 @@ static EVP_RAND_CTX *rand_get0_private(OSSL_LIB_CTX *ctx, RAND_GLOBAL *dgbl)
             return NULL;
         rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL,
                              SECONDARY_RESEED_TIME_INTERVAL);
-        CRYPTO_THREAD_set_local_ex(CRYPTO_THREAD_LOCAL_DRBG_PRIV_KEY, ctx, rand);
+        if (!CRYPTO_THREAD_set_local_ex(CRYPTO_THREAD_LOCAL_DRBG_PRIV_KEY, ctx, rand)) {
+            EVP_RAND_CTX_free(rand);
+            rand = NULL;
+        }
     }
     return rand;
 }