From: slontis <shane.lontis@oracle.com>
Date: Mon, 26 Aug 2024 01:14:55 +0000 (+1000)
Subject: Update code to use EVP_MD_xof()
X-Git-Tag: openssl-3.4.0-alpha1~34
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=976dd3581a00c5006bd696ac9ba7289de4d137d5;p=thirdparty%2Fopenssl.git

Update code to use EVP_MD_xof()

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25285)
---

diff --git a/apps/dgst.c b/apps/dgst.c
index 118754c4db6..818139f4e17 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -419,7 +419,7 @@ int dgst_main(int argc, char **argv)
         md_name = EVP_MD_get0_name(md);
 
     if (xoflen > 0) {
-        if (!(EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF)) {
+        if (!EVP_MD_xof(md)) {
             BIO_printf(bio_err, "Length can only be specified for XOF\n");
             goto end;
         }
diff --git a/apps/speed.c b/apps/speed.c
index 48f91b2213f..0079fd7c308 100644
--- a/apps/speed.c
+++ b/apps/speed.c
@@ -631,7 +631,7 @@ static int EVP_Digest_loop(const char *mdname, ossl_unused int algindex, void *a
 
     if (!opt_md_silent(mdname, &md))
         return -1;
-    if (EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) {
+    if (EVP_MD_xof(md)) {
         ctx = EVP_MD_CTX_new();
         if (ctx == NULL) {
             count = -1;
diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 86a81499c20..25fdfa53ee0 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -78,11 +78,11 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
 
 #ifdef FIPS_MODULE
     /* XOF are approved as standalone; Shake256 in Ed448; MGF */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
         return 0;
     }
-    if ((EVP_MD_get_flags(mgf1md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(mgf1md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
         return 0;
     }
@@ -196,11 +196,11 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
 
 #ifdef FIPS_MODULE
     /* XOF are approved as standalone; Shake256 in Ed448; MGF */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
         return -1;
     }
-    if ((EVP_MD_get_flags(mgf1md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(mgf1md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
         return -1;
     }
@@ -360,7 +360,7 @@ int PKCS1_MGF1(unsigned char *mask, long len,
     if (c == NULL)
         goto err;
     mdlen = EVP_MD_get_size(dgst);
-    if (mdlen < 0)
+    if (mdlen <= 0)
         goto err;
     /* step 4 */
     for (i = 0; outlen < len; i++) {
diff --git a/providers/implementations/exchange/dh_exch.c b/providers/implementations/exchange/dh_exch.c
index 2c054bb9921..b105826438d 100644
--- a/providers/implementations/exchange/dh_exch.c
+++ b/providers/implementations/exchange/dh_exch.c
@@ -392,7 +392,7 @@ static int dh_set_ctx_params(void *vpdhctx, const OSSL_PARAM params[])
         if (pdhctx->kdf_md == NULL)
             return 0;
         /* XOF digests are not allowed */
-        if ((EVP_MD_get_flags(pdhctx->kdf_md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(pdhctx->kdf_md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/exchange/ecdh_exch.c b/providers/implementations/exchange/ecdh_exch.c
index 39307e4e1e9..29ef20330d4 100644
--- a/providers/implementations/exchange/ecdh_exch.c
+++ b/providers/implementations/exchange/ecdh_exch.c
@@ -315,7 +315,7 @@ int ecdh_set_ctx_params(void *vpecdhctx, const OSSL_PARAM params[])
         if (pectx->kdf_md == NULL)
             return 0;
         /* XOF digests are not allowed */
-        if ((EVP_MD_get_flags(pectx->kdf_md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(pectx->kdf_md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/hmacdrbg_kdf.c b/providers/implementations/kdfs/hmacdrbg_kdf.c
index 3df5221580a..bdaea6b4a27 100644
--- a/providers/implementations/kdfs/hmacdrbg_kdf.c
+++ b/providers/implementations/kdfs/hmacdrbg_kdf.c
@@ -217,7 +217,7 @@ static int hmac_drbg_kdf_set_ctx_params(void *vctx,
         /* Confirm digest is allowed. Allow all digests that are not XOF */
         md = ossl_prov_digest_md(&drbg->digest);
         if (md != NULL) {
-            if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+            if (EVP_MD_xof(md)) {
                 ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
                 return 0;
             }
diff --git a/providers/implementations/kdfs/pbkdf2.c b/providers/implementations/kdfs/pbkdf2.c
index 1c94ece4942..0615aecfa98 100644
--- a/providers/implementations/kdfs/pbkdf2.c
+++ b/providers/implementations/kdfs/pbkdf2.c
@@ -266,7 +266,7 @@ static int kdf_pbkdf2_set_ctx_params(void *vctx, const OSSL_PARAM params[])
         if (!ossl_prov_digest_load_from_params(&ctx->digest, params, provctx))
             return 0;
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/sshkdf.c b/providers/implementations/kdfs/sshkdf.c
index c1d3066951c..191b7be0c5f 100644
--- a/providers/implementations/kdfs/sshkdf.c
+++ b/providers/implementations/kdfs/sshkdf.c
@@ -229,7 +229,7 @@ static int kdf_sshkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[])
             return 0;
 
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/sskdf.c b/providers/implementations/kdfs/sskdf.c
index ce9d0a71505..ff5ec50138f 100644
--- a/providers/implementations/kdfs/sskdf.c
+++ b/providers/implementations/kdfs/sskdf.c
@@ -577,7 +577,7 @@ static int sskdf_common_set_ctx_params(KDF_SSKDF *ctx, const OSSL_PARAM params[]
             return 0;
 
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/tls1_prf.c b/providers/implementations/kdfs/tls1_prf.c
index 451d8a818f8..31316401bc6 100644
--- a/providers/implementations/kdfs/tls1_prf.c
+++ b/providers/implementations/kdfs/tls1_prf.c
@@ -324,7 +324,7 @@ static int kdf_tls1_prf_set_ctx_params(void *vctx, const OSSL_PARAM params[])
             return 0;
 
         md = ossl_prov_digest_md(&digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             ossl_prov_digest_reset(&digest);
             return 0;
diff --git a/providers/implementations/kdfs/x942kdf.c b/providers/implementations/kdfs/x942kdf.c
index 41eaf52404d..205738cef13 100644
--- a/providers/implementations/kdfs/x942kdf.c
+++ b/providers/implementations/kdfs/x942kdf.c
@@ -517,7 +517,7 @@ static int x942kdf_set_ctx_params(void *vctx, const OSSL_PARAM params[])
         if (!ossl_prov_digest_load_from_params(&ctx->digest, params, provctx))
             return 0;
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/rands/drbg.c b/providers/implementations/rands/drbg.c
index 825fe30214f..255bf9b5074 100644
--- a/providers/implementations/rands/drbg.c
+++ b/providers/implementations/rands/drbg.c
@@ -1030,7 +1030,7 @@ int ossl_drbg_verify_digest(PROV_DRBG *drbg, OSSL_LIB_CTX *libctx,
     }
 #else   /* FIPS_MODULE */
     /* Outside of FIPS, any digests that are not XOF are allowed */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
         return 0;
     }
diff --git a/providers/implementations/signature/dsa_sig.c b/providers/implementations/signature/dsa_sig.c
index a4b8c21f76f..ec2205aa5aa 100644
--- a/providers/implementations/signature/dsa_sig.c
+++ b/providers/implementations/signature/dsa_sig.c
@@ -156,7 +156,7 @@ static int dsa_setup_md(PROV_DSA_CTX *ctx,
             goto err;
         }
         /* XOF digests don't work */
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             goto err;
         }
diff --git a/providers/implementations/signature/ecdsa_sig.c b/providers/implementations/signature/ecdsa_sig.c
index 1a58850b23f..46d100995c5 100644
--- a/providers/implementations/signature/ecdsa_sig.c
+++ b/providers/implementations/signature/ecdsa_sig.c
@@ -289,7 +289,7 @@ static int ecdsa_setup_md(PROV_ECDSA_CTX *ctx, const char *mdname,
         goto err;
     }
     /* XOF digests don't work */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
         goto err;
     }
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 2ca9c658985..45c36899e4c 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -397,7 +397,7 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
          * We don't support XOF digests with RSA PSS (yet), so just fail.
          * When we do support them, uncomment the second clause.
          */
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0
+        if (EVP_MD_xof(md)
                 /* && ctx->pad_mode != RSA_PKCS1_PSS_PADDING */) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             goto err;
diff --git a/providers/implementations/signature/sm2_sig.c b/providers/implementations/signature/sm2_sig.c
index 5172d6622be..3decca33c48 100644
--- a/providers/implementations/signature/sm2_sig.c
+++ b/providers/implementations/signature/sm2_sig.c
@@ -99,7 +99,7 @@ static int sm2sig_set_mdname(PROV_SM2_CTX *psm2ctx, const char *mdname)
         return 0;
 
     /* XOF digests don't work */
-    if ((EVP_MD_get_flags(psm2ctx->md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(psm2ctx->md)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
         return 0;
     }
diff --git a/test/evp_test.c b/test/evp_test.c
index 6a47765745d..69ecafe7eb9 100644
--- a/test/evp_test.c
+++ b/test/evp_test.c
@@ -788,7 +788,7 @@ static int digest_test_run(EVP_TEST *t)
         goto err;
     }
 
-    xof |= (EVP_MD_get_flags(expected->digest) & EVP_MD_FLAG_XOF) != 0;
+    xof |= EVP_MD_xof(expected->digest);
     if (xof) {
         EVP_MD_CTX *mctx_cpy;