Add test for EVP_KEYMGMT leak in evp_pkey_signature_init() error paths

Verify that calling EVP_PKEY_sign_init_ex2() with a mismatched
key/signature algorithm (RSA key with ECDSA signature) does not leak
EVP_KEYMGMT references. The test repeats the operation 100 times so
that ASAN can detect accumulating leaks.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Eugene Syromiatnikov <esyr@openssl.org>
MergeDate: Fri Feb 13 14:42:40 2026
(Merged from https://github.com/openssl/openssl/pull/29810)

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 7d1c8c585b134d884de308cddc40a8db5097e807..70f17a9d0b65162e97b7d6a390ad6898c5ba35a0 100644 (file)
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -5807,6 +5807,52 @@ end:
     return testresult;
 }
 
+#ifndef OPENSSL_NO_EC
+/*
+ * Test that EVP_PKEY_sign_init_ex2() with a mismatched key/signature algorithm
+ * (e.g. RSA key with ECDSA signature) correctly fails.
+ */
+static int test_EVP_PKEY_sign_init_mismatched_key_alg(void)
+{
+    EVP_PKEY *rsa_key = NULL;
+    EVP_PKEY_CTX *ctx = NULL;
+    EVP_PKEY_CTX *pkey_ctx = NULL;
+    EVP_SIGNATURE *ecdsa_sig = NULL;
+    int testresult = 0;
+
+    /* Generate an RSA key */
+    if (!TEST_ptr(pkey_ctx = EVP_PKEY_CTX_new_from_name(testctx, "RSA", NULL))
+        || !TEST_int_gt(EVP_PKEY_keygen_init(pkey_ctx), 0)
+        || !TEST_int_gt(EVP_PKEY_CTX_set_rsa_keygen_bits(pkey_ctx, 2048), 0)
+        || !TEST_int_gt(EVP_PKEY_keygen(pkey_ctx, &rsa_key), 0))
+        goto end;
+
+    EVP_PKEY_CTX_free(pkey_ctx);
+    pkey_ctx = NULL;
+
+    /* Fetch ECDSA signature algorithm - incompatible with RSA key */
+    if (!TEST_ptr(ecdsa_sig = EVP_SIGNATURE_fetch(testctx, "ECDSA", NULL)))
+        goto end;
+
+    /* Try to init sign with mismatched key/algorithm - this should fail */
+    if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_pkey(testctx, rsa_key, NULL)))
+        goto end;
+
+    /* This should fail with -2 (operation not supported for key type) */
+    if (!TEST_int_eq(EVP_PKEY_sign_init_ex2(ctx, ecdsa_sig, NULL), -2))
+        goto end;
+
+    testresult = 1;
+
+end:
+    EVP_PKEY_CTX_free(ctx);
+    EVP_PKEY_CTX_free(pkey_ctx);
+    EVP_SIGNATURE_free(ecdsa_sig);
+    EVP_PKEY_free(rsa_key);
+    return testresult;
+}
+#endif
+
 static int aes_gcm_encrypt(const unsigned char *gcm_key, size_t gcm_key_s,
     const unsigned char *gcm_iv, size_t gcm_ivlen,
     const unsigned char *gcm_pt, size_t gcm_pt_s,
@@ -6714,6 +6760,9 @@ int setup_tests(void)
     ADD_ALL_TESTS(test_EVP_PKEY_sign, 3);
 #ifndef OPENSSL_NO_DEPRECATED_3_0
     ADD_ALL_TESTS(test_EVP_PKEY_sign_with_app_method, 2);
+#endif
+#ifndef OPENSSL_NO_EC
+    ADD_TEST(test_EVP_PKEY_sign_init_mismatched_key_alg);
 #endif
     ADD_ALL_TESTS(test_EVP_Enveloped, 2);
     ADD_ALL_TESTS(test_d2i_AutoPrivateKey, OSSL_NELEM(keydata));