rsa-oaep: block SHAKE usage in FIPS mode

NIST SP 800-56 rev2 only allows using approved hash algorithms in
OAEP. Unlike FIPS 186-5 it doesn't have text allowing to use XOF SHAKE
functions. Maybe future revisions of SP 800-56 will adopt similar text
to FIPS 186-5 and allow XOF as MD and MGF (not MGF1).

RFC documents do not specify if SHAKE is allowed or blocked for usage
(i.e. there is no equivalent of RFC 8692 or RFC 8702 for OAEP). Status
quo allows their usage.

Add test cases for SHAKE in RSA-OAEP as allowed in default provider,
and blocked in fips.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/24387)

diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index b9030440c45087d0341ed8c409ca00421e69718b..0ec18b969119964543e48a17dc55c3d7abd7ad47 100644 (file)
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -76,6 +76,18 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
     if (mgf1md == NULL)
         mgf1md = md;
 
+#ifdef FIPS_MODULE
+    /* XOF are approved as standalone; Shake256 in Ed448; MGF */
+    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
+        return 0;
+    }
+    if ((EVP_MD_get_flags(mgf1md) & EVP_MD_FLAG_XOF) != 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
+        return 0;
+    }
+#endif
+
     mdlen = EVP_MD_get_size(md);
     if (mdlen <= 0) {
         ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_LENGTH);
@@ -182,6 +194,18 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
     if (mgf1md == NULL)
         mgf1md = md;
 
+#ifdef FIPS_MODULE
+    /* XOF are approved as standalone; Shake256 in Ed448; MGF */
+    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
+        return -1;
+    }
+    if ((EVP_MD_get_flags(mgf1md) & EVP_MD_FLAG_XOF) != 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
+        return -1;
+    }
+#endif
+
     mdlen = EVP_MD_get_size(md);
 
     if (tlen <= 0 || flen <= 0)

diff --git a/test/recipes/30-test_evp_data/evppkey_rsa_common.txt b/test/recipes/30-test_evp_data/evppkey_rsa_common.txt
index 0036acdb0f9f2079212b13a74df72092d89a389c..29f9f03aee1e2a56d7edbfb8476febe2b00ce5af 100644 (file)
--- a/test/recipes/30-test_evp_data/evppkey_rsa_common.txt
+++ b/test/recipes/30-test_evp_data/evppkey_rsa_common.txt
@@ -328,6 +328,38 @@ Ctrl = rsa_padding_mode:oaep
 Input = 458708DFBD42A1297CE7A9C86C7087AB80B1754810929B89C5107CA55368587686986FCE94D86CC1595B3FB736223A656EC0F34D18BA1CC5665593610F56C58E26B272D584F3D983A5C91085700755AEBD921FB280BBA3EDA7046EC07B43E7298E52D59EDC92BE4639A8CE08B2F85976ECF6D98CC469EEB9D5D8E2A32EA8A6626EDAFE1038B3DF455668A9F3C77CAD8B92FB872E00058C3D2A7EDE1A1F03FC5622084AE04D9D24F6BF0995C58D35B93B699B9763595E123F2AB0863CC9229EB290E2EDE7715C7A8F39E0B9A3E2E1B56EBB62F1CBFBB5986FB212EBD785B83D01D968B11D1756C7337F70C1F1A63BFF03608E24F3A2FD44E67F832A8701C5D5AF
 Output = "Hello World"
 
+# Decrypt OAEP SHAKE MGF1
+Availablein = default
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:oaep
+Ctrl = rsa_mgf1_md:shake128
+Input = 5998271817ECB28F061CA7A036CF8B79FCFA69C5C5FD782EB6B76AAA137338E3FB4F09BC3E53002EDE3B279F7470071F22C088B56D06626992EB5A20CA596B2F61BF7691CC1F9AD18ACC1E4E8598000A7829F387024740C42DF89D83A9C5032BB891964405019D0144A251B13F9FC6A652B5FF07697541F5A616AC7D39BA8F47CC2D48BC327201C8A0A0E0543BC7ACB0A99545DB94C8EE94D928D813AD06E4B2790AF30B42BFB141516BA1D3285C0A46E2EC0ABD23A3BB19556A58E4EEBB35950464C8C961C9707C66AE580CBD64583482A6B88FE1B68CFFE5A0AE5797DFAB90D3F594C624917EC25ADBFDEEBF212C9C15C6AE4AFA35A4F3CF79FC4BD3F8A91A
+Output = "Hello World"
+
+# Decrypt OAEP SHAKE MD
+Availablein = default
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:oaep
+Ctrl = rsa_oaep_md:shake128
+Input = 9DA9DA65D80395D1BB6A410F08B5A6B7D911463BE5804894DCE78973995DB98654E873CEFE001C436E15091D3E95137B51B84099553C97E144A102CF4A3BC9EB48036827F1F77E9308A34BC335EBF2FC6A1F0C6DB910BE75222109F9D3D2A02683F680DDD5C08D2A3273A90607F34E6454D72FD39095128775906A6B3064F2E14122A5FB8BF874DAB27D65B637A38AF93C68609699F180EE0DE551D8B31A90B5934632AA2C770F667D3C59917CC4D32E0F964E9516728E87974B0D072598B4E027A4CA3D80E8979677C40A1F7391481C81EE3CB0A529138F1E1D41538E5A06D7CB08C49F01F601EED40784ABCFC3A9C4BB8BA557277DE6FAF9F08EF092446CE0
+Output = "Hello World"
+
+# Decrypt OAEP SHAKE MGF1
+Availablein = fips
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:oaep
+Ctrl = rsa_mgf1_md:shake128
+Input = 5998271817ECB28F061CA7A036CF8B79FCFA69C5C5FD782EB6B76AAA137338E3FB4F09BC3E53002EDE3B279F7470071F22C088B56D06626992EB5A20CA596B2F61BF7691CC1F9AD18ACC1E4E8598000A7829F387024740C42DF89D83A9C5032BB891964405019D0144A251B13F9FC6A652B5FF07697541F5A616AC7D39BA8F47CC2D48BC327201C8A0A0E0543BC7ACB0A99545DB94C8EE94D928D813AD06E4B2790AF30B42BFB141516BA1D3285C0A46E2EC0ABD23A3BB19556A58E4EEBB35950464C8C961C9707C66AE580CBD64583482A6B88FE1B68CFFE5A0AE5797DFAB90D3F594C624917EC25ADBFDEEBF212C9C15C6AE4AFA35A4F3CF79FC4BD3F8A91A
+Result = KEYOP_ERROR
+
+# Decrypt OAEP SHAKE MD
+Availablein = fips
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:oaep
+Ctrl = rsa_oaep_md:shake128
+Input = 9DA9DA65D80395D1BB6A410F08B5A6B7D911463BE5804894DCE78973995DB98654E873CEFE001C436E15091D3E95137B51B84099553C97E144A102CF4A3BC9EB48036827F1F77E9308A34BC335EBF2FC6A1F0C6DB910BE75222109F9D3D2A02683F680DDD5C08D2A3273A90607F34E6454D72FD39095128775906A6B3064F2E14122A5FB8BF874DAB27D65B637A38AF93C68609699F180EE0DE551D8B31A90B5934632AA2C770F667D3C59917CC4D32E0F964E9516728E87974B0D072598B4E027A4CA3D80E8979677C40A1F7391481C81EE3CB0A529138F1E1D41538E5A06D7CB08C49F01F601EED40784ABCFC3A9C4BB8BA557277DE6FAF9F08EF092446CE0
+Result = KEYOP_ERROR
+
 # OAEP padding, corrupted ciphertext
 Decrypt = RSA-2048
 Ctrl = rsa_padding_mode:oaep