Add test for CMS decryption with RSA keys

Ensure we don't encounter a segfault when decrypting CMS messages with
malformed EnvelopedData when using RSA-OAEP.

Co-authored-by: Tomas Mraz <tomas@openssl.foundation>
Reviewed-by: Saša Nedvědický <sashan@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.foundation>
MergeDate: Mon Apr  6 18:58:32 2026
(cherry picked from commit 02986569b897f7ed8370f7fe751f0fdfe1ae8946)

diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index 4cfd953ccd80604fcf2791ad07ba1285ff2cba0d..fae89c8797c40d933c7846048d6c5c8ee02da7db 100644 (file)
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -56,7 +56,7 @@ my ($no_des, $no_dh, $no_dsa, $no_ec, $no_ec2m, $no_rc2, $no_zlib)
 
 $no_rc2 = 1 if disabled("legacy");
 
-plan tests => 36;
+plan tests => 37;
 
 ok(run(test(["pkcs7_test"])), "test pkcs7");
 
@@ -1407,6 +1407,18 @@ with({ exit_checker => sub { return shift == 4; } },
         }
     });
 
+$smcont_malformed = srctop_file("test", "recipes", "80-test_cms_data", "rsa-malformed.der");
+my $smrsacert = catfile($smdir, "smrsa3.pem");
+my $smrsakey = catfile($smdir, "smrsa3-key.pem");
+
+# Test case for CVE-2026-28390
+with({ exit_checker => sub { return shift == 4; } },
+    sub {
+        ok(run(app(["openssl", "cms", @prov, "-decrypt", "-in", $smcont_malformed, "-inform",
+                   "DER", "-recip", $smrsacert, "-inkey", $smrsakey, "-out", "{output}.cms"])),
+           "Must not crash on malformed cms inputs with RSA key");
+    });
+
 # Test encrypt to three recipients, and decrypt using key-only;
 # i.e. do not follow the recommended practice of providing the
 # recipient cert in the decrypt op.

diff --git a/test/recipes/80-test_cms_data/rsa-malformed.der b/test/recipes/80-test_cms_data/rsa-malformed.der
new file mode 100644 (file)
index 0000000..4182a46
Binary files /dev/null and b/test/recipes/80-test_cms_data/rsa-malformed.der differ