cms: Fix no-signed-attributes for unknown hashless algorithms

Fix CMS signing without signed-attributes for unknown (provider-supplied)
algorithms with don't operate on a digest (e.g. Falcon).

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
MergeDate: Tue Mar 10 18:25:39 2026
(Merged from https://github.com/openssl/openssl/pull/30287)

(cherry picked from commit 28c271365a9ef1bcdc7839ec2113cc56ed63d68d)

diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c
index 6598f73b8ef440527d99299d9eb748b483be8bdb..afca47a703dfda44bb9c0f27311fa66d9834e55c 100644 (file)
--- a/crypto/cms/cms_sd.c
+++ b/crypto/cms/cms_sd.c
@@ -480,11 +480,16 @@ static const struct {
 static const char *cms_mdless_signing(EVP_PKEY *pkey)
 {
     unsigned int i;
+    int def_nid = NID_undef;
 
     for (i = 0; key2data[i].name != NULL; i++) {
         if (EVP_PKEY_is_a(pkey, key2data[i].name))
             return key2data[i].name;
     }
+    if (EVP_PKEY_get_default_digest_nid(pkey, &def_nid) <= 0) {
+        /* Key doesn't have default digest, it's mdless */
+        return EVP_PKEY_get0_type_name(pkey);
+    }
     return NULL;
 }
 
@@ -553,7 +558,11 @@ static int ossl_cms_adjust_md(EVP_PKEY *pk, const EVP_MD **md, unsigned int flag
         return 1;
     }
 
+    if (*md != NULL)
+        (void)ERR_set_mark(); /* No error if no default md and user-supplied md is set */
     tmp_md = ossl_cms_get_default_md(pk, &md_a_must);
+    if (*md != NULL)
+        (void)ERR_pop_to_mark();
     if (md_a_must)
         *md = tmp_md;
     else if (*md == NULL)