Fix PKINIT CMS error checking for older OpenSSL

Commit 70f61d417261ca17efe3d60d180033bea2da60b0 updated the
CMS_verify() error code checks, using two error codes new to OpenSSL
3.0 (RSA_R_DIGEST_NOT_ALLOWED and CMS_R_UNKNOWN_DIGEST_ALGORITHM).
This change broke the build for OpenSSL 1.0 and 1.1.

Instead of looking for codes indicating an algorithm issue and
assuming that everything else is an invalid signature, check for the
code indicating an invalid signature and assume that everything else
is an algorithm issue.

(cherry picked from commit e48e2e56a05a47fd932a941ac82c1131ceed47d0)

ticket: 9069
version_fixed: 1.20.2

diff --git a/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c b/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c
index 4f295109b4d8d71626b68c6d03055bc96283787d..a3afbc553624fbb307bfeebc8d345fcbae50ffb4 100644 (file)
--- a/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c
+++ b/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c
@@ -2102,18 +2102,10 @@ cms_signeddata_verify(krb5_context context,
             goto cleanup;
         out = BIO_new(BIO_s_mem());
         if (CMS_verify(cms, NULL, store, NULL, out, flags) == 0) {
-            unsigned long err = ERR_peek_last_error();
-            switch(ERR_GET_REASON(err)) {
-            case RSA_R_DIGEST_NOT_ALLOWED:
-            case CMS_R_UNKNOWN_DIGEST_ALGORITHM:
-            case CMS_R_NO_MATCHING_DIGEST:
-            case CMS_R_NO_MATCHING_SIGNATURE:
-                retval = KRB5KDC_ERR_DIGEST_IN_SIGNED_DATA_NOT_ACCEPTED;
-                break;
-            case CMS_R_VERIFICATION_FAILURE:
-            default:
+            if (ERR_peek_last_error() == CMS_R_VERIFICATION_FAILURE)
                 retval = KRB5KDC_ERR_INVALID_SIG;
-            }
+            else
+                retval = KRB5KDC_ERR_DIGEST_IN_SIGNED_DATA_NOT_ACCEPTED;
             (void)oerr(context, retval, _("Failed to verify CMS message"));
             goto cleanup;
         }